01HQ49S7P2P295S281HKPEDEAB: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HQ49SJV18FCFRF07XS78CWST

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-20T22:38:51.277Zjob dependencies complete; ready to run (waiting for 16 m 35 s)
22024-02-20T22:39:46.818Zjob assigned to worker 01HQ4AR2BNAB79ZAD0YD901KKA (queued for 55 s)
32024-02-20T22:39:47.403Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-20T22:40:10.995Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-20T22:40:10.995Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-20T22:40:33.588Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-20T22:40:33.588Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-20T22:40:50.955Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-20T22:40:50.955Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-20T22:41:07.327Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-20T22:41:07.333Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-20T22:41:07.344Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-20T22:41:07.344Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-20T22:41:07.363Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-20T22:41:07.363Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-20T22:41:07.393Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-20T22:41:07.393Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-20T22:41:07.401Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-20T22:41:07.401Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-20T22:41:07.430Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-20T22:41:07.431Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-20T22:41:07.467Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-20T22:41:07.467Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-20T22:41:07.497Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-20T22:41:07.567Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-20T22:41:30.862Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-20T22:41:30.862Zstarting task 0: "setup"
282024-02-20T22:41:31.132Z++ uname -s
292024-02-20T22:41:31.145Z+ kern=SunOS
302024-02-20T22:41:31.146Z+ case "$kern" in
312024-02-20T22:41:31.146Z+ groupadd -g 12345 build
322024-02-20T22:41:31.146Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-20T22:41:33.143Z+ zfs create -o mountpoint=/work rpool/work
342024-02-20T22:41:33.234Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-20T22:41:33.241Z+ home_fs=zfs
362024-02-20T22:41:33.241Z+ [[ zfs == autofs ]]
372024-02-20T22:41:33.241Z+ mkdir -p /home/build
382024-02-20T22:41:33.242Z+ chown build:build /home/build /work
392024-02-20T22:41:35.239Z+ chmod 0700 /home/build /work
402024-02-20T22:41:35.253Zprocess exited: duration 4481 ms, exit code 0
 
412024-02-20T22:41:35.293Zstarting task 1: "authentication"
422024-02-20T22:41:35.310Zprocess exited: duration 29 ms, exit code 0
 
432024-02-20T22:41:35.336Zstarting task 2: "build"
442024-02-20T22:41:35.345Z+ banner cores
452024-02-20T22:41:35.411Z
462024-02-20T22:41:35.411Z #### #### ##### ###### ####
472024-02-20T22:41:35.411Z # # # # # # # #
482024-02-20T22:41:35.411Z # # # # # ##### ####
492024-02-20T22:41:35.411Z # # # ##### # #
502024-02-20T22:41:35.411Z # # # # # # # # #
512024-02-20T22:41:35.411Z #### #### # # ###### ####
522024-02-20T22:41:35.411Z
532024-02-20T22:41:35.421Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-20T22:41:35.421Z+ echo 'input bins dir contains:'
552024-02-20T22:41:35.421Zinput bins dir contains:
562024-02-20T22:41:35.421Z+ ls -ltr /input/build/work/bins
572024-02-20T22:41:35.439Ztotal 740414
582024-02-20T22:41:35.439Z-rw-r--r-- 1 root root 109656643 Feb 20 22:40 crucible-downstairs.gz
592024-02-20T22:41:35.439Z-rw-r--r-- 1 root root 91462952 Feb 20 22:40 crucible-hammer.gz
602024-02-20T22:41:35.439Z-rw-r--r-- 1 root root 105607799 Feb 20 22:40 crutest.gz
612024-02-20T22:41:35.439Z-rw-r--r-- 1 root root 72121864 Feb 20 22:41 dsc.gz
622024-02-20T22:41:35.439Z+ echo 'input script dir contains:'
632024-02-20T22:41:35.440Zinput script dir contains:
642024-02-20T22:41:35.440Z+ ls -ltr /input/build/work/scripts
652024-02-20T22:41:35.440Ztotal 39
662024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 3056 Feb 20 22:41 perf-downstairs-tick.d
672024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 1738 Feb 20 22:41 test_ds.sh
682024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 3746 Feb 20 22:41 test_live_repair.sh
692024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 6416 Feb 20 22:41 test_repair.sh
702024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 2969 Feb 20 22:41 test_replay.sh
712024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 9465 Feb 20 22:41 test_up.sh
722024-02-20T22:41:35.440Z-rw-r--r-- 1 root root 2824 Feb 20 22:41 upstairs_info.d
732024-02-20T22:41:35.440Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-20T22:41:35.440Z+ echo ' chmod input script dir contains:'
752024-02-20T22:41:35.440Z chmod input script dir contains:
762024-02-20T22:41:35.440Z+ ls -ltr /input/build/work/scripts
772024-02-20T22:41:35.440Ztotal 39
782024-02-20T22:41:35.440Z-rwxr-xr-x 1 root root 3056 Feb 20 22:41 perf-downstairs-tick.d
792024-02-20T22:41:35.440Z-rwxr-xr-x 1 root root 1738 Feb 20 22:41 test_ds.sh
802024-02-20T22:41:35.440Z-rwxr-xr-x 1 root root 3746 Feb 20 22:41 test_live_repair.sh
812024-02-20T22:41:35.440Z-rwxr-xr-x 1 root root 6416 Feb 20 22:41 test_repair.sh
822024-02-20T22:41:35.440Z-rwxr-xr-x 1 root root 2969 Feb 20 22:41 test_replay.sh
832024-02-20T22:41:35.441Z-rwxr-xr-x 1 root root 9465 Feb 20 22:41 test_up.sh
842024-02-20T22:41:35.441Z-rwxr-xr-x 1 root root 2824 Feb 20 22:41 upstairs_info.d
852024-02-20T22:41:35.441Z+ banner unpack
862024-02-20T22:41:35.441Z
872024-02-20T22:41:35.441Z # # # # ##### ## #### # #
882024-02-20T22:41:35.441Z # # ## # # # # # # # # #
892024-02-20T22:41:35.441Z # # # # # # # # # # ####
902024-02-20T22:41:35.441Z # # # # # ##### ###### # # #
912024-02-20T22:41:35.441Z # # # ## # # # # # # #
922024-02-20T22:41:35.441Z #### # # # # # #### # #
932024-02-20T22:41:35.441Z
942024-02-20T22:41:35.441Z+ mkdir -p /var/tmp/bins
952024-02-20T22:41:35.441Z+ for t in "$input/bins/"*.gz
962024-02-20T22:41:35.461Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-20T22:41:35.461Z+ b=crucible-downstairs.gz
982024-02-20T22:41:35.461Z+ b=crucible-downstairs
992024-02-20T22:41:35.461Z+ gunzip
1002024-02-20T22:41:38.004Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-20T22:41:38.017Z+ for t in "$input/bins/"*.gz
1022024-02-20T22:41:38.017Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-20T22:41:38.017Z+ b=crucible-hammer.gz
1042024-02-20T22:41:38.017Z+ b=crucible-hammer
1052024-02-20T22:41:38.017Z+ gunzip
1062024-02-20T22:41:40.115Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-20T22:41:40.132Z+ for t in "$input/bins/"*.gz
1082024-02-20T22:41:40.132Z++ basename /input/build/work/bins/crutest.gz
1092024-02-20T22:41:40.133Z+ b=crutest.gz
1102024-02-20T22:41:40.133Z+ b=crutest
1112024-02-20T22:41:40.133Z+ gunzip
1122024-02-20T22:41:42.819Z+ chmod +x /var/tmp/bins/crutest
1132024-02-20T22:41:42.820Z+ for t in "$input/bins/"*.gz
1142024-02-20T22:41:42.820Z++ basename /input/build/work/bins/dsc.gz
1152024-02-20T22:41:42.820Z+ b=dsc.gz
1162024-02-20T22:41:42.820Z+ b=dsc
1172024-02-20T22:41:42.820Z+ gunzip
1182024-02-20T22:41:44.638Z+ chmod +x /var/tmp/bins/dsc
1192024-02-20T22:41:45.311Z+ export BINDIR=/var/tmp/bins
1202024-02-20T22:41:45.311Z+ BINDIR=/var/tmp/bins
1212024-02-20T22:41:45.311Z+ banner repair
1222024-02-20T22:41:45.312Z
1232024-02-20T22:41:45.312Z ##### ###### ##### ## # #####
1242024-02-20T22:41:45.312Z # # # # # # # # # #
1252024-02-20T22:41:45.312Z # # ##### # # # # # # #
1262024-02-20T22:41:45.312Z ##### # ##### ###### # #####
1272024-02-20T22:41:45.312Z # # # # # # # # #
1282024-02-20T22:41:45.312Z # # ###### # # # # # #
1292024-02-20T22:41:45.312Z
1302024-02-20T22:41:45.313Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-20T22:41:45.313ZTurn off color for downstairs dump
1322024-02-20T22:41:45.313ZCreating region directory at: /var/tmp/test_repair
1332024-02-20T22:41:45.313ZCreating dsc directory at: /tmp/dsc
1342024-02-20T22:41:45.313ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.06319306
1352024-02-20T22:41:45.313ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.03917708
1362024-02-20T22:41:45.313ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.037988126
1372024-02-20T22:41:45.314ZRegion set with 3 regions was created
1382024-02-20T22:41:45.314Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-20T22:41:45.314ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-20T22:41:45.314Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.067547387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141}
1412024-02-20T22:41:45.314Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.06859491Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141}
1422024-02-20T22:41:45.314Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.068614389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141}
1432024-02-20T22:41:45.314Z{"msg":"Crucible 8434abc4-f25f-4fa5-838a-c6bced53ad9e has session id: 846c86f5-4ef2-4af2-9af7-f42fb3dc85e0","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.068629119Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1442024-02-20T22:41:45.314Z{"msg":"Upstairs opts: Upstairs UUID: 8434abc4-f25f-4fa5-838a-c6bced53ad9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.068647468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1452024-02-20T22:41:45.315Z{"msg":"Crucible stats registered with UUID: 8434abc4-f25f-4fa5-838a-c6bced53ad9e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.068662457Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1462024-02-20T22:41:45.315ZCrucible runtime is spawned
1472024-02-20T22:41:45.315Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069515177Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141}
1482024-02-20T22:41:45.315Z{"msg":"8434abc4-f25f-4fa5-838a-c6bced53ad9e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069634703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1492024-02-20T22:41:45.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069670021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1502024-02-20T22:41:45.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069740149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1512024-02-20T22:41:45.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069778467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1522024-02-20T22:41:45.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069817606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1532024-02-20T22:41:45.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069854295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1542024-02-20T22:41:45.316Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069894263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1552024-02-20T22:41:45.316Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069931322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1562024-02-20T22:41:45.316Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.069963821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1572024-02-20T22:41:45.316Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.070104606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1582024-02-20T22:41:45.316Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.07026851Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1592024-02-20T22:41:45.316Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.070335907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1602024-02-20T22:41:45.316Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.070392185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1612024-02-20T22:41:45.316Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.071807905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1622024-02-20T22:41:45.316Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.071845353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1632024-02-20T22:41:45.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.071861663Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1642024-02-20T22:41:45.317Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.071889712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1652024-02-20T22:41:45.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.07193983Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1662024-02-20T22:41:45.317Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.071970979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1672024-02-20T22:41:45.317Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.072967073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1682024-02-20T22:41:45.317Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073121178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1692024-02-20T22:41:45.317Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073195065Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1702024-02-20T22:41:45.318Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073558512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1712024-02-20T22:41:45.318Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.07360937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1722024-02-20T22:41:45.318Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073642849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1732024-02-20T22:41:45.318Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073669638Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1742024-02-20T22:41:45.318Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073711267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1752024-02-20T22:41:45.318Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073751485Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1762024-02-20T22:41:45.318Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073786604Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1772024-02-20T22:41:45.318Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.073816143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1782024-02-20T22:41:45.318Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074079353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1792024-02-20T22:41:45.319Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074108583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1802024-02-20T22:41:45.319Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074275477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1812024-02-20T22:41:45.319Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074303205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1822024-02-20T22:41:45.319Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074318215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1832024-02-20T22:41:45.319Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074340264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1842024-02-20T22:41:45.319Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074355184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1852024-02-20T22:41:45.319Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074376243Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1862024-02-20T22:41:45.319Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074390732Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1872024-02-20T22:41:45.320Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074403892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1882024-02-20T22:41:45.320Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074417721Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1892024-02-20T22:41:45.320Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074437461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1902024-02-20T22:41:45.320Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074548497Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1912024-02-20T22:41:45.320Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074575966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1922024-02-20T22:41:45.320Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074589285Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1932024-02-20T22:41:45.320Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1942024-02-20T22:41:45.320Z,"time":"2024-02-20T22:41:44.074609494Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1952024-02-20T22:41:45.321Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074624554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1962024-02-20T22:41:45.321Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074644563Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1972024-02-20T22:41:45.321Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074657873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1982024-02-20T22:41:45.321Z{"msg":"8434abc4-f25f-4fa5-838a-c6bced53ad9e is now active with session: 846c86f5-4ef2-4af2-9af7-f42fb3dc85e0","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074677792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
1992024-02-20T22:41:45.321Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074690522Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141}
2002024-02-20T22:41:45.321Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:44.074708021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141,"session_id":"846c86f5-4ef2-4af2-9af7-f42fb3dc85e0"}
2012024-02-20T22:41:45.321ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-20T22:41:45.321ZFill test
2032024-02-20T22:41:45.332ZRead and Verify all blocks (0..600 range:false)
2042024-02-20T22:41:45.332ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-20T22:41:45.333ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-20T22:41:45.333Z----------------------------------------------------------------
2072024-02-20T22:41:45.333Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-20T22:41:45.333ZStates: Active Active Active
2092024-02-20T22:41:45.334ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-20T22:41:45.334Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-20T22:41:45.334Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-20T22:41:45.334Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-20T22:41:45.334Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-20T22:41:45.335Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-20T22:41:45.335Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-20T22:41:45.335Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-20T22:41:45.335Z New 0 0 0 0
2182024-02-20T22:41:45.335Z Sent 0 0 0 0
2192024-02-20T22:41:45.335Z Done 6 6 6 18
2202024-02-20T22:41:45.335Z Skipped 0 0 0 0
2212024-02-20T22:41:45.336Z Error 0 0 0 0
2222024-02-20T22:41:45.336ZLast Flush: 1006 1006 1006
2232024-02-20T22:41:45.336ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-20T22:41:45.336ZUpstairs last five completed: 13 12 11 10 9
2252024-02-20T22:41:45.336ZCLIENT: Up:0 ds:6 act:3
2262024-02-20T22:41:49.162Z----------------------------------------------------------------
2272024-02-20T22:41:49.163Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-20T22:41:49.163ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-20T22:41:49.163ZUpstairs last five completed: 14 13 12 11 10
2302024-02-20T22:41:49.163ZCLIENT: Up:0 ds:0 act:3
2312024-02-20T22:41:49.163ZCLIENT: All crucible jobs finished, exiting program
2322024-02-20T22:41:49.177Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:41:48.705502447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1141}
2332024-02-20T22:41:49.178Z
2342024-02-20T22:41:49.178ZBegin loop 1
2352024-02-20T22:41:49.178ZDownstairs to restart: 0
2362024-02-20T22:41:49.187Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.728420797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143}
2372024-02-20T22:41:49.187Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.728731796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143}
2382024-02-20T22:41:49.188Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.728757335Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143}
2392024-02-20T22:41:49.222Z{"msg":"Crucible c10997c5-f806-4b87-8700-9fbfc8cdaed6 has session id: c1c8cb33-38e2-40cd-a5dd-b48c47e0163f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.728771925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2402024-02-20T22:41:49.222Z{"msg":"Upstairs opts: Upstairs UUID: c10997c5-f806-4b87-8700-9fbfc8cdaed6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.728814233Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2412024-02-20T22:41:49.222Z{"msg":"Crucible stats registered with UUID: c10997c5-f806-4b87-8700-9fbfc8cdaed6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.728851652Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2422024-02-20T22:41:49.222ZCrucible runtime is spawned
2432024-02-20T22:41:49.222Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729050365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143}
2442024-02-20T22:41:49.222Z{"msg":"c10997c5-f806-4b87-8700-9fbfc8cdaed6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729081134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2452024-02-20T22:41:49.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729095703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2462024-02-20T22:41:49.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729113133Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2472024-02-20T22:41:49.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729146351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2482024-02-20T22:41:49.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.72917312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2492024-02-20T22:41:49.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729203739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2502024-02-20T22:41:49.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729230358Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2512024-02-20T22:41:49.223Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729275137Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2522024-02-20T22:41:49.223Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729313395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2532024-02-20T22:41:49.223Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729340324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2542024-02-20T22:41:49.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729488369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2552024-02-20T22:41:49.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729551267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2562024-02-20T22:41:49.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.729582736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2572024-02-20T22:41:49.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730204503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2582024-02-20T22:41:49.223Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730261781Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2592024-02-20T22:41:49.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.73031Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2602024-02-20T22:41:49.223Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730325989Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2612024-02-20T22:41:49.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730361668Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2622024-02-20T22:41:49.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730406096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2632024-02-20T22:41:49.224Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730905888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2642024-02-20T22:41:49.224Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.730943167Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2652024-02-20T22:41:49.224Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731019844Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2662024-02-20T22:41:49.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731459569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2672024-02-20T22:41:49.224Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731494837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2682024-02-20T22:41:49.224Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731510527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2692024-02-20T22:41:49.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731578804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2702024-02-20T22:41:49.225Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731605263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2712024-02-20T22:41:49.225Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731619483Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2722024-02-20T22:41:49.225Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731638912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2732024-02-20T22:41:49.225Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731652791Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2742024-02-20T22:41:49.225Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731674311Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2752024-02-20T22:41:49.225Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.73168906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2762024-02-20T22:41:49.225Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731710289Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2772024-02-20T22:41:49.225Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731724639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2782024-02-20T22:41:49.226Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731745508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2792024-02-20T22:41:49.226Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731772267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2802024-02-20T22:41:49.226Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731786947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2812024-02-20T22:41:49.226ZWait for a query_work_queue command to finish before sending IO
2822024-02-20T22:41:49.226Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731811476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2832024-02-20T22:41:49.226Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731826425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2842024-02-20T22:41:49.226Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731840605Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2852024-02-20T22:41:49.227Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731861174Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2862024-02-20T22:41:49.227Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731874754Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2872024-02-20T22:41:49.227Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731894913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2882024-02-20T22:41:49.227Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731908653Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2892024-02-20T22:41:49.227Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731927662Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2902024-02-20T22:41:49.227Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731941301Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2912024-02-20T22:41:49.227Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731961191Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2922024-02-20T22:41:49.227Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.73197508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2932024-02-20T22:41:49.228Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.731994879Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2942024-02-20T22:41:49.228Z{"msg":"c10997c5-f806-4b87-8700-9fbfc8cdaed6 is now active with session: c1c8cb33-38e2-40cd-a5dd-b48c47e0163f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.732017849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2952024-02-20T22:41:49.228Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.732049937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143,"session_id":"c1c8cb33-38e2-40cd-a5dd-b48c47e0163f"}
2962024-02-20T22:41:49.228Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:48.732075016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143}
2972024-02-20T22:41:49.228ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2982024-02-20T22:41:49.228ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-02-20T22:41:49.228ZRun Repair workload
3002024-02-20T22:41:49.228Z01/30 Write block 149 len 20480 data: 2 2 2 2 2
3012024-02-20T22:41:49.229Z02/30 Write block 388 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3022024-02-20T22:41:49.229Z03/30 Read block 154 len 20480
3032024-02-20T22:41:49.229Z04/30 Read block 417 len 16384
3042024-02-20T22:41:49.229Z05/30 Read block 82 len 32768
3052024-02-20T22:41:49.655Z06/30 Write block 452 len 28672 data: 2 2 2 2 2 2 2
3062024-02-20T22:41:49.655Z07/30 Read block 63 len 20480
3072024-02-20T22:41:49.655Z08/30 Read block 155 len 20480
3082024-02-20T22:41:49.655Z09/30 Read block 206 len 20480
3092024-02-20T22:41:49.655Z10/30 Read block 588 len 12288
3102024-02-20T22:41:49.655Z11/30 Write block 388 len 32768 data: 3 3 3 3 3 3 3 3
3112024-02-20T22:41:49.655Z12/30 Write block 462 len 36864 data: 2 2 2 2 2 2 2 2 2
3122024-02-20T22:41:49.656Z13/30 Read block 147 len 12288
3132024-02-20T22:41:49.656Z14/30 Read block 322 len 4096
3142024-02-20T22:41:49.656Z15/30 Read block 510 len 20480
3152024-02-20T22:41:49.656Z16/30 Read block 121 len 4096
3162024-02-20T22:41:49.656Z17/30 Write block 303 len 20480 data: 2 2 2 2 2
3172024-02-20T22:41:49.656Z18/30 Write block 363 len 24576 data: 2 2 2 2 2 2
3182024-02-20T22:41:49.656Z19/30 Read block 183 len 36864
3192024-02-20T22:41:49.657Z20/30 Write block 240 len 4096 data: 2
3202024-02-20T22:41:49.657Z21/30 Write block 332 len 8192 data: 2 2
3212024-02-20T22:41:49.657Z22/30 Read block 268 len 16384
3222024-02-20T22:41:49.657Z23/30 Flush
3232024-02-20T22:41:49.686Z24/30 Flush
3242024-02-20T22:41:49.686Z25/30 Write block 112 len 24576 data: 2 2 2 2 2 2
3252024-02-20T22:41:49.686Z26/30 Write block 538 len 32768 data: 2 2 2 2 2 2 2 2
3262024-02-20T22:41:49.686Z27/30 Write block 575 len 36864 data: 2 2 2 2 2 2 2 2 2
3272024-02-20T22:41:49.686Z28/30 Write block 410 len 28672 data: 2 2 2 2 2 2 2
3282024-02-20T22:41:49.686Z29/30 Read block 478 len 24576
3292024-02-20T22:41:49.686Z30/30 Read block 454 len 12288
3302024-02-20T22:41:49.686Z----------------------------------------------------------------
3312024-02-20T22:41:49.686Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-02-20T22:41:49.687ZStates: Active Active Active
3332024-02-20T22:41:49.687ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-02-20T22:41:49.687Z 1 Acked 1000 Write 5 Done Done Done false
3352024-02-20T22:41:49.687Z 2 Acked 1001 Write 10 Sent Done Done false
3362024-02-20T22:41:49.687Z 3 Acked 1002 Read 5 Sent Done Done false
3372024-02-20T22:41:49.687Z 4 Acked 1003 Read 4 Sent Done Done false
3382024-02-20T22:41:49.687Z 5 Acked 1004 Read 8 Sent Done Done false
3392024-02-20T22:41:49.688Z 6 Acked 1005 Write 7 Sent Done Done false
3402024-02-20T22:41:49.688Z 7 Acked 1006 Read 5 Sent Done Done false
3412024-02-20T22:41:49.688Z 8 Acked 1007 Read 5 Sent Done Done false
3422024-02-20T22:41:49.688Z 9 Acked 1008 Read 5 Sent Done Done false
3432024-02-20T22:41:49.688Z 10 Acked 1009 Read 3 Sent Done Done false
3442024-02-20T22:41:49.688Z 11 Acked 1010 Write 8 Sent Done Done false
3452024-02-20T22:41:49.688Z 12 Acked 1011 Write 9 Sent Done Done false
3462024-02-20T22:41:49.688Z 13 Acked 1012 Read 3 Sent Done Done false
3472024-02-20T22:41:49.688Z 14 Acked 1013 Read 1 Sent Done Done false
3482024-02-20T22:41:49.688Z 15 Acked 1014 Read 5 Sent Done Done false
3492024-02-20T22:41:49.689Z 16 Acked 1015 Read 1 Sent Done Done false
3502024-02-20T22:41:49.689Z 17 Acked 1016 Write 5 Sent Done Done false
3512024-02-20T22:41:49.689Z 18 Acked 1017 Write 6 Sent Done Done false
3522024-02-20T22:41:49.689Z 19 Acked 1018 Read 9 Sent Done Done false
3532024-02-20T22:41:49.689Z 20 Acked 1019 Write 1 Sent Done Done false
3542024-02-20T22:41:49.689Z 21 Acked 1020 Write 2 Sent Done Done false
3552024-02-20T22:41:49.689Z 22 Acked 1021 Read 4 Sent Done Done false
3562024-02-20T22:41:49.689Z 23 Acked 1022 Flush 0 Sent Done Done false
3572024-02-20T22:41:49.689Z 24 Acked 1023 Flush 0 Sent Done Done false
3582024-02-20T22:41:49.689Z 25 Acked 1024 Write 6 Sent Done Done false
3592024-02-20T22:41:49.689Z 26 Acked 1025 Write 8 Sent Done Done false
3602024-02-20T22:41:49.689Z 27 Acked 1026 Write 9 Sent Done Done false
3612024-02-20T22:41:49.690Z 28 Acked 1027 Write 7 Sent Done Done false
3622024-02-20T22:41:49.690Z 29 Acked 1028 Read 6 Sent Done Done false
3632024-02-20T22:41:49.690Z 30 Acked 1029 Read 3 Sent Done Done false
3642024-02-20T22:41:49.690Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-02-20T22:41:49.690Z New 0 0 0 0
3662024-02-20T22:41:49.690Z Sent 29 0 0 29
3672024-02-20T22:41:49.690Z Done 1 30 30 61
3682024-02-20T22:41:49.690Z Skipped 0 0 0 0
3692024-02-20T22:41:49.690Z Error 0 0 0 0
3702024-02-20T22:41:49.691ZLast Flush: 0 1023 1023
3712024-02-20T22:41:49.691ZDownstairs last five completed:
3722024-02-20T22:41:49.691ZUpstairs last five completed: 30 29 28 27 26
3732024-02-20T22:41:49.691Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:41:48.816243445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1143}
3742024-02-20T22:41:49.691ZWrote out file "/tmp/test_repair_verify.data"
3752024-02-20T22:41:49.691Z
3762024-02-20T22:41:49.691ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-02-20T22:41:49.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-02-20T22:41:49.691Z 0 000-019 1 1 1 1 1 1 F F F
3792024-02-20T22:41:49.691Z 1 020-039 1 1 1 1 1 1 F F F
3802024-02-20T22:41:49.691Z 2 040-059 1 1 1 1 1 1 F F F
3812024-02-20T22:41:49.692Z 3 060-079 1 1 1 1 1 1 F F F
3822024-02-20T22:41:49.692Z 4 080-099 1 1 1 1 1 1 F F F
3832024-02-20T22:41:49.692Z 5 100-119 1 1 1 1 1 1 F T T <---
3842024-02-20T22:41:49.692Z 6 120-139 1 1 1 1 1 1 F F F
3852024-02-20T22:41:49.692Z 7 140-159 1 2 2 1 2 2 T F F <---
3862024-02-20T22:41:49.692Z 8 160-179 1 1 1 1 1 1 F F F
3872024-02-20T22:41:49.692Z 9 180-199 1 1 1 1 1 1 F F F
3882024-02-20T22:41:49.692Z 10 200-219 1 1 1 1 1 1 F F F
3892024-02-20T22:41:49.692Z 11 220-239 1 1 1 1 1 1 F F F
3902024-02-20T22:41:49.692Z 12 240-259 1 2 2 1 2 2 F F F <---
3912024-02-20T22:41:49.692Z 13 260-279 1 1 1 1 1 1 F F F
3922024-02-20T22:41:49.692Z 14 280-299 1 1 1 1 1 1 F F F
3932024-02-20T22:41:49.692Z 15 300-319 1 2 2 1 2 2 F F F <---
3942024-02-20T22:41:49.693Z 16 320-339 1 2 2 1 2 2 F F F <---
3952024-02-20T22:41:49.693Z 17 340-359 1 1 1 1 1 1 F F F
3962024-02-20T22:41:49.693Z 18 360-379 1 2 2 1 2 2 F F F <---
3972024-02-20T22:41:49.693Z 19 380-399 1 2 2 1 2 2 F F F <---
3982024-02-20T22:41:49.693Z 20 400-419 1 1 1 1 1 1 F T T <---
3992024-02-20T22:41:49.693Z 21 420-439 1 1 1 1 1 1 F F F
4002024-02-20T22:41:49.693Z 22 440-459 1 2 2 1 2 2 F F F <---
4012024-02-20T22:41:49.693Z 23 460-479 1 2 2 1 2 2 F F F <---
4022024-02-20T22:41:49.693Z 24 480-499 1 1 1 1 1 1 F F F
4032024-02-20T22:41:49.693Z 25 500-519 1 1 1 1 1 1 F F F
4042024-02-20T22:41:49.694Z 26 520-539 1 1 1 1 1 1 F T T <---
4052024-02-20T22:41:49.694Z 27 540-559 1 1 1 1 1 1 F T T <---
4062024-02-20T22:41:49.694Z 28 560-579 1 1 1 1 1 1 F T T <---
4072024-02-20T22:41:49.694Z 29 580-599 1 1 1 1 1 1 F T T <---
4082024-02-20T22:41:49.694ZMax gen: 2, Max flush: 2
4092024-02-20T22:41:49.694ZError: Difference in extent metadata found!
4102024-02-20T22:41:49.694ZA Difference in extent metadata is expected here
4112024-02-20T22:41:49.694ZOn loop 1
4122024-02-20T22:41:49.694Z
4132024-02-20T22:41:49.694ZVerifying data now
4142024-02-20T22:41:55.285ZNo change after verify
4152024-02-20T22:41:55.285ZLoop: 1 Downstairs dump after verify (and repair):
4162024-02-20T22:41:55.299ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-02-20T22:41:55.299Z 0 000-019 1 1 1 1 1 1 F F F
4182024-02-20T22:41:55.299Z 1 020-039 1 1 1 1 1 1 F F F
4192024-02-20T22:41:55.299Z 2 040-059 1 1 1 1 1 1 F F F
4202024-02-20T22:41:55.299Z 3 060-079 1 1 1 1 1 1 F F F
4212024-02-20T22:41:55.299Z 4 080-099 1 1 1 1 1 1 F F F
4222024-02-20T22:41:55.299Z 5 100-119 3 3 3 3 3 3 F F F
4232024-02-20T22:41:55.311Z 6 120-139 1 1 1 1 1 1 F F F
4242024-02-20T22:41:55.311Z 7 140-159 2 2 2 2 2 2 F F F
4252024-02-20T22:41:55.311Z 8 160-179 1 1 1 1 1 1 F F F
4262024-02-20T22:41:55.311Z 9 180-199 1 1 1 1 1 1 F F F
4272024-02-20T22:41:55.311Z 10 200-219 1 1 1 1 1 1 F F F
4282024-02-20T22:41:55.311Z 11 220-239 1 1 1 1 1 1 F F F
4292024-02-20T22:41:55.311Z 12 240-259 2 2 2 2 2 2 F F F
4302024-02-20T22:41:55.311Z 13 260-279 1 1 1 1 1 1 F F F
4312024-02-20T22:41:55.311Z 14 280-299 1 1 1 1 1 1 F F F
4322024-02-20T22:41:55.311Z 15 300-319 2 2 2 2 2 2 F F F
4332024-02-20T22:41:55.311Z 16 320-339 2 2 2 2 2 2 F F F
4342024-02-20T22:41:55.311Z 17 340-359 1 1 1 1 1 1 F F F
4352024-02-20T22:41:55.311Z 18 360-379 2 2 2 2 2 2 F F F
4362024-02-20T22:41:55.311Z 19 380-399 2 2 2 2 2 2 F F F
4372024-02-20T22:41:55.311Z 20 400-419 3 3 3 3 3 3 F F F
4382024-02-20T22:41:55.311Z 21 420-439 1 1 1 1 1 1 F F F
4392024-02-20T22:41:55.311Z 22 440-459 2 2 2 2 2 2 F F F
4402024-02-20T22:41:55.311Z 23 460-479 2 2 2 2 2 2 F F F
4412024-02-20T22:41:55.312Z 24 480-499 1 1 1 1 1 1 F F F
4422024-02-20T22:41:55.312Z 25 500-519 1 1 1 1 1 1 F F F
4432024-02-20T22:41:55.312Z 26 520-539 3 3 3 3 3 3 F F F
4442024-02-20T22:41:55.312Z 27 540-559 3 3 3 3 3 3 F F F
4452024-02-20T22:41:55.312Z 28 560-579 3 3 3 3 3 3 F F F
4462024-02-20T22:41:55.312Z 29 580-599 3 3 3 3 3 3 F F F
4472024-02-20T22:41:55.312ZMax gen: 3, Max flush: 3
4482024-02-20T22:41:55.312Z
4492024-02-20T22:41:55.312ZBegin loop 2
4502024-02-20T22:41:55.312ZDownstairs to restart: 2
4512024-02-20T22:41:55.338Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.858823183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151}
4522024-02-20T22:41:55.338Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.85917195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151}
4532024-02-20T22:41:55.338Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859223899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151}
4542024-02-20T22:41:55.339Z{"msg":"Crucible 1f300eba-5368-43bb-a394-5085fa0d77b1 has session id: 587e6a1f-5c05-485f-b515-d8a024b4f650","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859251508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4552024-02-20T22:41:55.339ZCrucible runtime is spawned
4562024-02-20T22:41:55.339Z{"msg":"Upstairs opts: Upstairs UUID: 1f300eba-5368-43bb-a394-5085fa0d77b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859314985Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4572024-02-20T22:41:55.339Z{"msg":"Crucible stats registered with UUID: 1f300eba-5368-43bb-a394-5085fa0d77b1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859348904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4582024-02-20T22:41:55.339Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859366483Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151}
4592024-02-20T22:41:55.339Z{"msg":"1f300eba-5368-43bb-a394-5085fa0d77b1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859506529Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4602024-02-20T22:41:55.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859538777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4612024-02-20T22:41:55.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859558717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4622024-02-20T22:41:55.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859589326Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4632024-02-20T22:41:55.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859628974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4642024-02-20T22:41:55.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859649703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4652024-02-20T22:41:55.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859672003Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4662024-02-20T22:41:55.339Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859698702Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4672024-02-20T22:41:55.339Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859725691Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4682024-02-20T22:41:55.339Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859764819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4692024-02-20T22:41:55.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859808618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4702024-02-20T22:41:55.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.859955333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4712024-02-20T22:41:55.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860002161Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4722024-02-20T22:41:55.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860517652Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4732024-02-20T22:41:55.339Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860562021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4742024-02-20T22:41:55.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860602059Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4752024-02-20T22:41:55.339Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860648568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4762024-02-20T22:41:55.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860686236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4772024-02-20T22:41:55.339Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.860702316Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4782024-02-20T22:41:55.340Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861103861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4792024-02-20T22:41:55.340Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.86114197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4802024-02-20T22:41:55.340Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861339353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4812024-02-20T22:41:55.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861644902Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4822024-02-20T22:41:55.340Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.86170226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4832024-02-20T22:41:55.340Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861749548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4842024-02-20T22:41:55.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861775567Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4852024-02-20T22:41:55.340Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861808566Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4862024-02-20T22:41:55.340Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861824735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4872024-02-20T22:41:55.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861853614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4882024-02-20T22:41:55.340Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861876334Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4892024-02-20T22:41:55.340Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861934392Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4902024-02-20T22:41:55.340Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.861961071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4912024-02-20T22:41:55.340Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.86197633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4922024-02-20T22:41:55.340Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.86199038Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4932024-02-20T22:41:55.340Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862019228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4942024-02-20T22:41:55.340Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862035508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4952024-02-20T22:41:55.340Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862049327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4962024-02-20T22:41:55.340Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862079626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4972024-02-20T22:41:55.340Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4982024-02-20T22:41:55.340Z,"time":"2024-02-20T22:41:54.862104276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
4992024-02-20T22:41:55.340Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862150254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5002024-02-20T22:41:55.340Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862183363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5012024-02-20T22:41:55.341Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862199312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5022024-02-20T22:41:55.341Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862216432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5032024-02-20T22:41:55.341Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862240771Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5042024-02-20T22:41:55.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.8622629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5052024-02-20T22:41:55.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862295179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5062024-02-20T22:41:55.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862319708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5072024-02-20T22:41:55.341Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862349257Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5082024-02-20T22:41:55.341Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862362936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5092024-02-20T22:41:55.341Z{"msg":"1f300eba-5368-43bb-a394-5085fa0d77b1 is now active with session: 587e6a1f-5c05-485f-b515-d8a024b4f650","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862380586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5102024-02-20T22:41:55.341Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862413845Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151,"session_id":"587e6a1f-5c05-485f-b515-d8a024b4f650"}
5112024-02-20T22:41:55.341Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:41:54.862448603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151}
5122024-02-20T22:41:55.350ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5132024-02-20T22:41:55.350ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-02-20T22:41:55.351ZRun Repair workload
5152024-02-20T22:41:55.351Z01/30 Flush
5162024-02-20T22:41:55.351Z02/30 Write block 339 len 4096 data: 2
5172024-02-20T22:41:55.351Z03/30 Read block 131 len 40960
5182024-02-20T22:41:55.351Z04/30 Read block 497 len 28672
5192024-02-20T22:41:55.351Z05/30 Read block 86 len 32768
5202024-02-20T22:41:55.351Z06/30 Write block 277 len 32768 data: 2 2 2 2 2 2 2 2
5212024-02-20T22:41:55.351Z07/30 Flush
5222024-02-20T22:41:55.351Z08/30 Write block 263 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5232024-02-20T22:41:55.351Z09/30 Read block 419 len 24576
5242024-02-20T22:41:55.351Z10/30 Read block 165 len 8192
5252024-02-20T22:41:55.351Z11/30 Read block 549 len 36864
5262024-02-20T22:41:55.351Z12/30 Read block 107 len 32768
5272024-02-20T22:41:55.351Z13/30 Write block 461 len 36864 data: 2 3 3 3 3 3 3 3 3
5282024-02-20T22:41:55.351Z14/30 Write block 320 len 12288 data: 2 2 2
5292024-02-20T22:41:55.351Z15/30 Read block 368 len 28672
5302024-02-20T22:41:55.351Z16/30 Read block 538 len 36864
5312024-02-20T22:41:55.351Z17/30 Flush
5322024-02-20T22:41:55.370Z18/30 Write block 272 len 16384 data: 3 2 2 2
5332024-02-20T22:41:55.403Z19/30 Read block 200 len 16384
5342024-02-20T22:41:55.403Z20/30 Flush
5352024-02-20T22:41:55.403Z21/30 Flush
5362024-02-20T22:41:55.403Z22/30 Write block 28 len 32768 data: 2 2 2 2 2 2 2 2
5372024-02-20T22:41:55.403Z23/30 Read block 515 len 28672
5382024-02-20T22:41:55.403Z24/30 Write block 134 len 12288 data: 2 2 2
5392024-02-20T22:41:55.403Z25/30 Write block 69 len 8192 data: 2 2
5402024-02-20T22:41:55.420Z26/30 Write block 25 len 16384 data: 2 2 2 3
5412024-02-20T22:41:55.420Z27/30 Read block 456 len 40960
5422024-02-20T22:41:55.420Z28/30 Read block 172 len 36864
5432024-02-20T22:41:55.420Z29/30 Write block 535 len 4096 data: 2
5442024-02-20T22:41:55.420Z30/30 Write block 526 len 24576 data: 2 2 2 2 2 2
5452024-02-20T22:41:55.420Z----------------------------------------------------------------
5462024-02-20T22:41:55.420Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:29
5472024-02-20T22:41:55.420ZStates: Active Active Active
5482024-02-20T22:41:55.420ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-02-20T22:41:55.420Z 2 Acked 1001 Write 1 Done Done Done false
5502024-02-20T22:41:55.421Z 3 Acked 1002 Read 10 Done Done Done false
5512024-02-20T22:41:55.421Z 4 Acked 1003 Read 7 Done Done Done false
5522024-02-20T22:41:55.421Z 5 Acked 1004 Read 8 Done Done Done false
5532024-02-20T22:41:55.421Z 6 Acked 1005 Write 8 Done Done Sent false
5542024-02-20T22:41:55.421Z 7 Acked 1006 Flush 0 Done Done Sent false
5552024-02-20T22:41:55.421Z 8 Acked 1007 Write 10 Done Done Sent false
5562024-02-20T22:41:55.421Z 9 Acked 1008 Read 6 Done Done Sent false
5572024-02-20T22:41:55.421Z 10 Acked 1009 Read 2 Done Done Sent false
5582024-02-20T22:41:55.421Z 11 Acked 1010 Read 9 Done Done Sent false
5592024-02-20T22:41:55.421Z 12 Acked 1011 Read 8 Done Done Sent false
5602024-02-20T22:41:55.421Z 13 Acked 1012 Write 9 Done Done Sent false
5612024-02-20T22:41:55.421Z 14 Acked 1013 Write 3 Done Done Sent false
5622024-02-20T22:41:55.421Z 15 Acked 1014 Read 7 Done Done Sent false
5632024-02-20T22:41:55.421Z 16 Acked 1015 Read 9 Done Done Sent false
5642024-02-20T22:41:55.421Z 17 Acked 1016 Flush 0 Done Done Sent false
5652024-02-20T22:41:55.421Z 18 Acked 1017 Write 4 Done Done Sent false
5662024-02-20T22:41:55.421Z 19 Acked 1018 Read 4 Done Done Sent false
5672024-02-20T22:41:55.421Z 20 Acked 1019 Flush 0 Done Done Sent false
5682024-02-20T22:41:55.421Z 21 Acked 1020 Flush 0 Done Done Sent false
5692024-02-20T22:41:55.421Z 22 Acked 1021 Write 8 Done Done Sent false
5702024-02-20T22:41:55.421Z 23 Acked 1022 Read 7 Done Done Sent false
5712024-02-20T22:41:55.421Z 24 Acked 1023 Write 3 Done Done Sent false
5722024-02-20T22:41:55.421Z 25 Acked 1024 Write 2 Done Done Sent false
5732024-02-20T22:41:55.421Z 26 Acked 1025 Write 4 Done Done Sent false
5742024-02-20T22:41:55.421Z 27 Acked 1026 Read 10 Done Done Sent false
5752024-02-20T22:41:55.422Z 28 Acked 1027 Read 9 Done Done Sent false
5762024-02-20T22:41:55.422Z 29 Acked 1028 Write 1 Done Done Sent false
5772024-02-20T22:41:55.422Z 30 Acked 1029 Write 6 Sent Sent Sent false
5782024-02-20T22:41:55.422Z STATES DS:0 DS:1 DS:2 TOTAL
5792024-02-20T22:41:55.422Z New 0 0 0 0
5802024-02-20T22:41:55.422Z Sent 1 1 25 27
5812024-02-20T22:41:55.422Z Done 28 28 4 60
5822024-02-20T22:41:55.422Z Skipped 0 0 0 0
5832024-02-20T22:41:55.422Z Error 0 0 0 0
5842024-02-20T22:41:55.422ZLast Flush: 1020 1020 1000
5852024-02-20T22:41:55.422ZDownstairs last five completed: 1000
5862024-02-20T22:41:55.422ZUpstairs last five completed: 30 29 28 27 26
5872024-02-20T22:41:55.422Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:41:54.925261946Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1151}
5882024-02-20T22:41:55.422ZWrote out file "/tmp/test_repair_verify.data"
5892024-02-20T22:41:55.422Z
5902024-02-20T22:41:55.422ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5912024-02-20T22:41:55.422ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5922024-02-20T22:41:55.422Z 0 000-019 1 1 1 1 1 1 F F F
5932024-02-20T22:41:55.422Z 1 020-039 1 1 1 1 1 1 T T F <---
5942024-02-20T22:41:55.422Z 2 040-059 1 1 1 1 1 1 F F F
5952024-02-20T22:41:55.422Z 3 060-079 1 1 1 1 1 1 T T F <---
5962024-02-20T22:41:55.422Z 4 080-099 1 1 1 1 1 1 F F F
5972024-02-20T22:41:55.422Z 5 100-119 3 3 3 3 3 3 F F F
5982024-02-20T22:41:55.422Z 6 120-139 1 1 1 1 1 1 T T F <---
5992024-02-20T22:41:55.423Z 7 140-159 2 2 2 2 2 2 F F F
6002024-02-20T22:41:55.423Z 8 160-179 1 1 1 1 1 1 F F F
6012024-02-20T22:41:55.423Z 9 180-199 1 1 1 1 1 1 F F F
6022024-02-20T22:41:55.423Z 10 200-219 1 1 1 1 1 1 F F F
6032024-02-20T22:41:55.423Z 11 220-239 1 1 1 1 1 1 F F F
6042024-02-20T22:41:55.423Z 12 240-259 2 2 2 2 2 2 F F F
6052024-02-20T22:41:55.423Z 13 260-279 4 4 1 7 7 1 F F F <---
6062024-02-20T22:41:55.423Z 14 280-299 4 4 1 5 5 1 F F F <---
6072024-02-20T22:41:55.423Z 15 300-319 2 2 2 2 2 2 F F F
6082024-02-20T22:41:55.423Z 16 320-339 4 4 2 6 6 2 F F T <---
6092024-02-20T22:41:55.423Z 17 340-359 1 1 1 1 1 1 F F F
6102024-02-20T22:41:55.423Z 18 360-379 2 2 2 2 2 2 F F F
6112024-02-20T22:41:55.423Z 19 380-399 2 2 2 2 2 2 F F F
6122024-02-20T22:41:55.423Z 20 400-419 3 3 3 3 3 3 F F F
6132024-02-20T22:41:55.423Z 21 420-439 1 1 1 1 1 1 F F F
6142024-02-20T22:41:55.423Z 22 440-459 2 2 2 2 2 2 F F F
6152024-02-20T22:41:55.423Z 23 460-479 4 4 2 6 6 2 F F F <---
6162024-02-20T22:41:55.423Z 24 480-499 1 1 1 1 1 1 F F F
6172024-02-20T22:41:55.423Z 25 500-519 1 1 1 1 1 1 F F F
6182024-02-20T22:41:55.423Z 26 520-539 3 3 3 3 3 3 T T F <---
6192024-02-20T22:41:55.423Z 27 540-559 3 3 3 3 3 3 F F F
6202024-02-20T22:41:55.423Z 28 560-579 3 3 3 3 3 3 F F F
6212024-02-20T22:41:55.423Z 29 580-599 3 3 3 3 3 3 F F F
6222024-02-20T22:41:55.423ZMax gen: 4, Max flush: 7
6232024-02-20T22:41:55.423ZError: Difference in extent metadata found!
6242024-02-20T22:41:55.424ZA Difference in extent metadata is expected here
6252024-02-20T22:41:55.424ZOn loop 2
6262024-02-20T22:41:55.424Z
6272024-02-20T22:41:55.434ZVerifying data now
6282024-02-20T22:42:02.499ZNo change after verify
6292024-02-20T22:42:02.499ZLoop: 2 Downstairs dump after verify (and repair):
6302024-02-20T22:42:03.060ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6312024-02-20T22:42:03.060Z 0 000-019 1 1 1 1 1 1 F F F
6322024-02-20T22:42:03.060Z 1 020-039 5 5 5 8 8 8 F F F
6332024-02-20T22:42:03.060Z 2 040-059 1 1 1 1 1 1 F F F
6342024-02-20T22:42:03.060Z 3 060-079 5 5 5 8 8 8 F F F
6352024-02-20T22:42:03.060Z 4 080-099 1 1 1 1 1 1 F F F
6362024-02-20T22:42:03.060Z 5 100-119 3 3 3 3 3 3 F F F
6372024-02-20T22:42:03.060Z 6 120-139 5 5 5 8 8 8 F F F
6382024-02-20T22:42:03.060Z 7 140-159 2 2 2 2 2 2 F F F
6392024-02-20T22:42:03.060Z 8 160-179 1 1 1 1 1 1 F F F
6402024-02-20T22:42:03.060Z 9 180-199 1 1 1 1 1 1 F F F
6412024-02-20T22:42:03.060Z 10 200-219 1 1 1 1 1 1 F F F
6422024-02-20T22:42:03.060Z 11 220-239 1 1 1 1 1 1 F F F
6432024-02-20T22:42:03.060Z 12 240-259 2 2 2 2 2 2 F F F
6442024-02-20T22:42:03.060Z 13 260-279 4 4 4 7 7 7 F F F
6452024-02-20T22:42:03.060Z 14 280-299 4 4 4 5 5 5 F F F
6462024-02-20T22:42:03.060Z 15 300-319 2 2 2 2 2 2 F F F
6472024-02-20T22:42:03.060Z 16 320-339 4 4 4 6 6 6 F F F
6482024-02-20T22:42:03.060Z 17 340-359 1 1 1 1 1 1 F F F
6492024-02-20T22:42:03.060Z 18 360-379 2 2 2 2 2 2 F F F
6502024-02-20T22:42:03.060Z 19 380-399 2 2 2 2 2 2 F F F
6512024-02-20T22:42:03.061Z 20 400-419 3 3 3 3 3 3 F F F
6522024-02-20T22:42:03.061Z 21 420-439 1 1 1 1 1 1 F F F
6532024-02-20T22:42:03.061Z 22 440-459 2 2 2 2 2 2 F F F
6542024-02-20T22:42:03.061Z 23 460-479 4 4 4 6 6 6 F F F
6552024-02-20T22:42:03.061Z 24 480-499 1 1 1 1 1 1 F F F
6562024-02-20T22:42:03.061Z 25 500-519 1 1 1 1 1 1 F F F
6572024-02-20T22:42:03.061Z 26 520-539 5 5 5 8 8 8 F F F
6582024-02-20T22:42:03.061Z 27 540-559 3 3 3 3 3 3 F F F
6592024-02-20T22:42:03.061Z 28 560-579 3 3 3 3 3 3 F F F
6602024-02-20T22:42:03.061Z 29 580-599 3 3 3 3 3 3 F F F
6612024-02-20T22:42:03.061ZMax gen: 5, Max flush: 8
6622024-02-20T22:42:03.061Z
6632024-02-20T22:42:03.061ZBegin loop 3
6642024-02-20T22:42:03.061ZDownstairs to restart: 1
6652024-02-20T22:42:03.071Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.063674972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159}
6662024-02-20T22:42:03.071Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"Crucible runtime is spawned
6672024-02-20T22:42:03.071Zcrucible","level":30,"time":"2024-02-20T22:42:02.064074368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159}
6682024-02-20T22:42:03.071Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064108396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159}
6692024-02-20T22:42:03.071Z{"msg":"Crucible 1310a39c-0666-4bec-a274-3ad955d2938f has session id: d8276743-bc30-4a99-a273-6c62647ef35b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064145785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6702024-02-20T22:42:03.071Z{"msg":"Upstairs opts: Upstairs UUID: 1310a39c-0666-4bec-a274-3ad955d2938f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064186784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6712024-02-20T22:42:03.071Z{"msg":"Crucible stats registered with UUID: 1310a39c-0666-4bec-a274-3ad955d2938f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064220412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6722024-02-20T22:42:03.071Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064238372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159}
6732024-02-20T22:42:03.071Z{"msg":"1310a39c-0666-4bec-a274-3ad955d2938f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.06427479Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6742024-02-20T22:42:03.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064298569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6752024-02-20T22:42:03.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064336708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6762024-02-20T22:42:03.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064471633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6772024-02-20T22:42:03.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064536491Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6782024-02-20T22:42:03.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064607298Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6792024-02-20T22:42:03.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064639157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6802024-02-20T22:42:03.072Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064656547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6812024-02-20T22:42:03.072Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064691955Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6822024-02-20T22:42:03.072Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064710505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6832024-02-20T22:42:03.072Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064731404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6842024-02-20T22:42:03.072Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064765733Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6852024-02-20T22:42:03.072Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.064784952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6862024-02-20T22:42:03.072Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065173698Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6872024-02-20T22:42:03.072Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065210387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6882024-02-20T22:42:03.072Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065227176Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6892024-02-20T22:42:03.072Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065240816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6902024-02-20T22:42:03.072Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065254345Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6912024-02-20T22:42:03.072Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065275834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6922024-02-20T22:42:03.072Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065817625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6932024-02-20T22:42:03.072Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065860714Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6942024-02-20T22:42:03.072Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.065878493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6952024-02-20T22:42:03.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066331187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6962024-02-20T22:42:03.072Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066378015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6972024-02-20T22:42:03.072Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066442433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6982024-02-20T22:42:03.073Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066461702Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
6992024-02-20T22:42:03.073Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066493701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7002024-02-20T22:42:03.073Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.06651469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7012024-02-20T22:42:03.073Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066541359Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7022024-02-20T22:42:03.073Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066559639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7032024-02-20T22:42:03.073Z{"msg":"[0]R flush_numbers[0..12]: [1, 8, 1, 8, 1, 3, 8, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066588678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7042024-02-20T22:42:03.073Z{"msg":"[0]R generation[0..12]: [1, 5, 1, 5, 1, 3, 5, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066607357Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7052024-02-20T22:42:03.073Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066637186Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7062024-02-20T22:42:03.073Z{"msg":"[1]R flush_numbers[0..12]: [1, 8, 1, 8, 1, 3, 8, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
7072024-02-20T22:42:03.073Z,"time":"2024-02-20T22:42:02.066658385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7082024-02-20T22:42:03.073Z{"msg":"[1]R generation[0..12]: [1, 5, 1, 5, 1, 3, 5, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066688594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7092024-02-20T22:42:03.073Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066710593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7102024-02-20T22:42:03.073Z{"msg":"[2]R flush_numbers[0..12]: [1, 8, 1, 8, 1, 3, 8, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066732353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7112024-02-20T22:42:03.073Z{"msg":"[2]R generation[0..12]: [1, 5, 1, 5, 1, 3, 5, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066764811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7122024-02-20T22:42:03.073Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7132024-02-20T22:42:03.073Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066829689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7142024-02-20T22:42:03.073Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066853028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7152024-02-20T22:42:03.073Z{"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066879007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7162024-02-20T22:42:03.073Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066902376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7172024-02-20T22:42:03.073Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066930515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7182024-02-20T22:42:03.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066952554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7192024-02-20T22:42:03.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.066980004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7202024-02-20T22:42:03.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.067029362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7212024-02-20T22:42:03.073Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7222024-02-20T22:42:03.073ZAll required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.067059421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7232024-02-20T22:42:03.074Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.06709074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7242024-02-20T22:42:03.074Z{"msg":"1310a39c-0666-4bec-a274-3ad955d2938f is now active with session: d8276743-bc30-4a99-a273-6c62647ef35b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.067115739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7252024-02-20T22:42:03.074Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.067140948Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159,"session_id":"d8276743-bc30-4a99-a273-6c62647ef35b"}
7262024-02-20T22:42:03.074Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:02.067163197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159}
7272024-02-20T22:42:03.074ZLoading write count information from file "/tmp/test_repair_verify.data"
7282024-02-20T22:42:03.074ZRun Repair workload
7292024-02-20T22:42:03.074Z01/30 Write block 301 len 8192 data: 2 2
7302024-02-20T22:42:03.074Z02/30 Read block 47 len 28672
7312024-02-20T22:42:03.074Z03/30 Write block 398 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7322024-02-20T22:42:03.074Z04/30 Read block 210 len 4096
7332024-02-20T22:42:03.074Z05/30 Write block 342 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7342024-02-20T22:42:03.074Z06/30 Write block 307 len 16384 data: 3 2 2 2
7352024-02-20T22:42:03.074Z07/30 Read block 4 len 24576
7362024-02-20T22:42:03.074Z08/30 Read block 236 len 8192
7372024-02-20T22:42:03.074Z09/30 Write block 322 len 4096 data: 3
7382024-02-20T22:42:03.074Z10/30 Read block 405 len 36864
7392024-02-20T22:42:03.074Z11/30 Read block 137 len 28672
7402024-02-20T22:42:03.074Z12/30 Read block 322 len 32768
7412024-02-20T22:42:03.074Z13/30 Write block 104 len 8192 data: 2 2
7422024-02-20T22:42:03.074Z14/30 Write block 534 len 40960 data: 2 3 2 2 3 3 3 3 3 3
7432024-02-20T22:42:03.074Z15/30 Write block 359 len 32768 data: 2 2 2 2 3 3 3 3
7442024-02-20T22:42:03.074Z16/30 Flush
7452024-02-20T22:42:03.074Z17/30 Write block 145 len 4096 data: 2
7462024-02-20T22:42:03.074Z18/30 Write block 137 len 8192 data: 2 2
7472024-02-20T22:42:03.074Z19/30 Write block 16 len 8192 data: 2 2
7482024-02-20T22:42:03.074Z20/30 Read block 296 len 24576
7492024-02-20T22:42:03.075Z21/30 Read block 355 len 32768
7502024-02-20T22:42:03.075Z22/30 Read block 339 len 28672
7512024-02-20T22:42:03.075Z23/30 Read block 10 len 12288
7522024-02-20T22:42:03.075Z24/30 Write block 175 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7532024-02-20T22:42:03.075Z25/30 Read block 594 len 24576
7542024-02-20T22:42:03.075Z26/30 Flush
7552024-02-20T22:42:03.075Z27/30 Write block 165 len 24576 data: 2 2 2 2 2 2
7562024-02-20T22:42:03.075Z28/30 Read block 85 len 28672
7572024-02-20T22:42:03.075Z29/30 Read block 210 len 8192
7582024-02-20T22:42:03.075Z30/30 Read block 98 len 20480
7592024-02-20T22:42:03.075Z----------------------------------------------------------------
7602024-02-20T22:42:03.075Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7612024-02-20T22:42:03.075ZStates: Active Active Active
7622024-02-20T22:42:03.075ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7632024-02-20T22:42:03.075Z 1 Acked 1000 Write 2 Done Done Done false
7642024-02-20T22:42:03.075Z 2 Acked 1001 Read 7 Done Done Done false
7652024-02-20T22:42:03.084Z 3 Acked 1002 Write 10 Done Done Done false
7662024-02-20T22:42:03.084Z 4 Acked 1003 Read 1 Done Done Done false
7672024-02-20T22:42:03.084Z 5 Acked 1004 Write 10 Done Done Done false
7682024-02-20T22:42:03.084Z 6 Acked 1005 Write 4 Done Done Done false
7692024-02-20T22:42:03.084Z 7 Acked 1006 Read 6 Done Done Done false
7702024-02-20T22:42:03.084Z 8 Acked 1007 Read 2 Done Sent Done false
7712024-02-20T22:42:03.084Z 9 Acked 1008 Write 1 Done Sent Done false
7722024-02-20T22:42:03.084Z 10 Acked 1009 Read 9 Done Sent Done false
7732024-02-20T22:42:03.084Z 11 Acked 1010 Read 7 Done Sent Done false
7742024-02-20T22:42:03.084Z 12 Acked 1011 Read 8 Done Sent Done false
7752024-02-20T22:42:03.084Z 13 Acked 1012 Write 2 Done Sent Done false
7762024-02-20T22:42:03.084Z 14 Acked 1013 Write 10 Done Sent Done false
7772024-02-20T22:42:03.084Z 15 Acked 1014 Write 8 Done Sent Done false
7782024-02-20T22:42:03.084Z 16 Acked 1015 Flush 0 Done Sent Done false
7792024-02-20T22:42:03.084Z 17 Acked 1016 Write 1 Done Sent Done false
7802024-02-20T22:42:03.085Z 18 Acked 1017 Write 2 Done Sent Done false
7812024-02-20T22:42:03.085Z 19 Acked 1018 Write 2 Done Sent Done false
7822024-02-20T22:42:03.085Z 20 Acked 1019 Read 6 Done Sent Done false
7832024-02-20T22:42:03.085Z 21 Acked 1020 Read 8 Done Sent Done false
7842024-02-20T22:42:03.085Z 22 Acked 1021 Read 7 Done Sent Done false
7852024-02-20T22:42:03.085Z 23 Acked 1022 Read 3 Done Sent Done false
7862024-02-20T22:42:03.085Z 24 Acked 1023 Write 10 Done Sent Done false
7872024-02-20T22:42:03.085Z 25 Acked 1024 Read 6 Done Sent Done false
7882024-02-20T22:42:03.085Z 26 Acked 1025 Flush 0 Done Sent Done false
7892024-02-20T22:42:03.085Z 27 Acked 1026 Write 6 Done Sent Done false
7902024-02-20T22:42:03.085Z 28 Acked 1027 Read 7 Done Sent Done false
7912024-02-20T22:42:03.085Z 29 Acked 1028 Read 2 Done Sent Done false
7922024-02-20T22:42:03.085Z 30 Acked 1029 Read 5 Done Sent Done false
7932024-02-20T22:42:03.085Z STATES DS:0 DS:1 DS:2 TOTAL
7942024-02-20T22:42:03.085Z New 0 0 0 0
7952024-02-20T22:42:03.085Z Sent 0 23 0 23
7962024-02-20T22:42:03.085Z Done 30 7 30 67
7972024-02-20T22:42:03.085Z Skipped 0 0 0 0
7982024-02-20T22:42:03.085Z Error 0 0 0 0
7992024-02-20T22:42:03.085ZLast Flush: 1025 0 1025
8002024-02-20T22:42:03.085ZDownstairs last five completed:
8012024-02-20T22:42:03.085ZUpstairs last five completed: 30 29 28 27 26
8022024-02-20T22:42:03.085Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:02.136275834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1159}
8032024-02-20T22:42:03.085ZWrote out file "/tmp/test_repair_verify.data"
8042024-02-20T22:42:03.085Z
8052024-02-20T22:42:03.086ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8062024-02-20T22:42:03.086ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8072024-02-20T22:42:03.086Z 0 000-019 6 1 6 10 1 10 F F F <---
8082024-02-20T22:42:03.086Z 1 020-039 5 5 5 8 8 8 F F F
8092024-02-20T22:42:03.086Z 2 040-059 1 1 1 1 1 1 F F F
8102024-02-20T22:42:03.086Z 3 060-079 5 5 5 8 8 8 F F F
8112024-02-20T22:42:03.086Z 4 080-099 1 1 1 1 1 1 F F F
8122024-02-20T22:42:03.086Z 5 100-119 6 3 6 9 3 9 F F F <---
8132024-02-20T22:42:03.086Z 6 120-139 6 5 6 10 8 10 F F F <---
8142024-02-20T22:42:03.086Z 7 140-159 6 2 6 10 2 10 F F F <---
8152024-02-20T22:42:03.086Z 8 160-179 6 1 6 10 1 10 T F T <---
8162024-02-20T22:42:03.086Z 9 180-199 6 1 6 10 1 10 F F F <---
8172024-02-20T22:42:03.086Z 10 200-219 1 1 1 1 1 1 F F F
8182024-02-20T22:42:03.086Z 11 220-239 1 1 1 1 1 1 F F F
8192024-02-20T22:42:03.086Z 12 240-259 2 2 2 2 2 2 F F F
8202024-02-20T22:42:03.086Z 13 260-279 4 4 4 7 7 7 F F F
8212024-02-20T22:42:03.086Z 14 280-299 4 4 4 5 5 5 F F F
8222024-02-20T22:42:03.086Z 15 300-319 6 2 6 9 2 9 F T F <---
8232024-02-20T22:42:03.086Z 16 320-339 6 4 6 9 6 9 F F F <---
8242024-02-20T22:42:03.086Z 17 340-359 6 1 6 9 1 9 F T F <---
8252024-02-20T22:42:03.086Z 18 360-379 6 2 6 9 2 9 F F F <---
8262024-02-20T22:42:03.086Z 19 380-399 6 2 6 9 2 9 F T F <---
8272024-02-20T22:42:03.086Z 20 400-419 6 3 6 9 3 9 F T F <---
8282024-02-20T22:42:03.086Z 21 420-439 1 1 1 1 1 1 F F F
8292024-02-20T22:42:03.086Z 22 440-459 2 2 2 2 2 2 F F F
8302024-02-20T22:42:03.087Z 23 460-479 4 4 4 6 6 6 F F F
8312024-02-20T22:42:03.087Z 24 480-499 1 1 1 1 1 1 F F F
8322024-02-20T22:42:03.087Z 25 500-519 1 1 1 1 1 1 F F F
8332024-02-20T22:42:03.087Z 26 520-539 6 5 6 9 8 9 F F F <---
8342024-02-20T22:42:03.087Z 27 540-559 6 3 6 9 3 9 F F F <---
8352024-02-20T22:42:03.087Z 28 560-579 3 3 3 3 3 3 F F F
8362024-02-20T22:42:03.087Z 29 580-599 3 3 3 3 3 3 F F F
8372024-02-20T22:42:03.087ZMax gen: 6, Max flush: 10
8382024-02-20T22:42:03.087ZError: Difference in extent metadata found!
8392024-02-20T22:42:03.087ZA Difference in extent metadata is expected here
8402024-02-20T22:42:03.087ZOn loop 3
8412024-02-20T22:42:03.087Z
8422024-02-20T22:42:03.087ZVerifying data now
8432024-02-20T22:42:09.107ZNo change after verify
8442024-02-20T22:42:09.107ZLoop: 3 Downstairs dump after verify (and repair):
8452024-02-20T22:42:09.122ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8462024-02-20T22:42:09.122Z 0 000-019 6 6 6 10 10 10 F F F
8472024-02-20T22:42:09.122Z 1 020-039 5 5 5 8 8 8 F F F
8482024-02-20T22:42:09.122Z 2 040-059 1 1 1 1 1 1 F F F
8492024-02-20T22:42:09.122Z 3 060-079 5 5 5 8 8 8 F F F
8502024-02-20T22:42:09.122Z 4 080-099 1 1 1 1 1 1 F F F
8512024-02-20T22:42:09.122Z 5 100-119 6 6 6 9 9 9 F F F
8522024-02-20T22:42:09.122Z 6 120-139 6 6 6 10 10 10 F F F
8532024-02-20T22:42:09.123Z 7 140-159 6 6 6 10 10 10 F F F
8542024-02-20T22:42:09.123Z 8 160-179 7 7 7 11 11 11 F F F
8552024-02-20T22:42:10.360Z 9 180-199 6 6 6 10 10 10 F F F
8562024-02-20T22:42:10.360Z 10 200-219 1 1 1 1 1 1 F F F
8572024-02-20T22:42:10.360Z 11 220-239 1 1 1 1 1 1 F F F
8582024-02-20T22:42:10.360Z 12 240-259 2 2 2 2 2 2 F F F
8592024-02-20T22:42:10.361Z 13 260-279 4 4 4 7 7 7 F F F
8602024-02-20T22:42:10.361Z 14 280-299 4 4 4 5 5 5 F F F
8612024-02-20T22:42:10.361Z 15 300-319 6 6 6 9 9 9 F F F
8622024-02-20T22:42:10.361Z 16 320-339 6 6 6 9 9 9 F F F
8632024-02-20T22:42:10.362Z 17 340-359 6 6 6 9 9 9 F F F
8642024-02-20T22:42:10.362Z 18 360-379 6 6 6 9 9 9 F F F
8652024-02-20T22:42:10.362Z 19 380-399 6 6 6 9 9 9 F F F
8662024-02-20T22:42:10.362Z 20 400-419 6 6 6 9 9 9 F F F
8672024-02-20T22:42:10.363Z 21 420-439 1 1 1 1 1 1 F F F
8682024-02-20T22:42:10.363Z 22 440-459 2 2 2 2 2 2 F F F
8692024-02-20T22:42:10.363Z 23 460-479 4 4 4 6 6 6 F F F
8702024-02-20T22:42:10.363Z 24 480-499 1 1 1 1 1 1 F F F
8712024-02-20T22:42:10.364Z 25 500-519 1 1 1 1 1 1 F F F
8722024-02-20T22:42:10.364Z 26 520-539 6 6 6 9 9 9 F F F
8732024-02-20T22:42:10.364Z 27 540-559 6 6 6 9 9 9 F F F
8742024-02-20T22:42:10.364Z 28 560-579 3 3 3 3 3 3 F F F
8752024-02-20T22:42:10.364Z 29 580-599 3 3 3 3 3 3 F F F
8762024-02-20T22:42:10.364ZMax gen: 7, Max flush: 11
8772024-02-20T22:42:10.365Z
8782024-02-20T22:42:10.365ZBegin loop 4
8792024-02-20T22:42:10.365ZDownstairs to restart: 1
8802024-02-20T22:42:10.380Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.679971331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167}
8812024-02-20T22:42:10.380Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680227592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167}
8822024-02-20T22:42:10.381Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680244841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167}
8832024-02-20T22:42:10.381Z{"msg":"Crucible d0ecb075-3dd5-4e4a-8fcf-03c681ed830a has session id: f2bc9059-9d6d-446f-a691-ac6458b15d40","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.68027431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8842024-02-20T22:42:10.381Z{Crucible runtime is spawned
8852024-02-20T22:42:10.382Z"msg":"Upstairs opts: Upstairs UUID: d0ecb075-3dd5-4e4a-8fcf-03c681ed830a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680329048Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8862024-02-20T22:42:10.382Z{"msg":"Crucible stats registered with UUID: d0ecb075-3dd5-4e4a-8fcf-03c681ed830a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680368537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8872024-02-20T22:42:10.382Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680430655Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167}
8882024-02-20T22:42:10.382Z{"msg":"d0ecb075-3dd5-4e4a-8fcf-03c681ed830a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680480093Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8892024-02-20T22:42:10.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680505812Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8902024-02-20T22:42:10.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.68054381Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8912024-02-20T22:42:10.383Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.68057039Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8922024-02-20T22:42:10.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680596858Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8932024-02-20T22:42:10.383Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680634057Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8942024-02-20T22:42:10.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680657916Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8952024-02-20T22:42:10.384Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680686215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8962024-02-20T22:42:10.384Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680715624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8972024-02-20T22:42:10.384Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680742203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8982024-02-20T22:42:10.384Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680890198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
8992024-02-20T22:42:10.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680931377Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9002024-02-20T22:42:10.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.680958325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9012024-02-20T22:42:10.385Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.681489717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9022024-02-20T22:42:10.385Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.681518545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9032024-02-20T22:42:10.385Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.681533495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9042024-02-20T22:42:10.385Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.681548355Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9052024-02-20T22:42:10.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.681562064Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9062024-02-20T22:42:10.386Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.681607812Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9072024-02-20T22:42:10.386Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682126504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9082024-02-20T22:42:10.386Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682178842Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9092024-02-20T22:42:10.386Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.6822329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9102024-02-20T22:42:10.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682698493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9112024-02-20T22:42:10.387Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682737712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9122024-02-20T22:42:10.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682764731Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9132024-02-20T22:42:10.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.68280293Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9142024-02-20T22:42:10.387Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682829358Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9152024-02-20T22:42:10.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682858248Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9162024-02-20T22:42:10.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682880787Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9172024-02-20T22:42:10.388Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682909006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9182024-02-20T22:42:10.388Z{"msg":"[0]R flush_numbers[0..12]: [10, 8, 1, 8, 1, 9, 10, 10, 11, 10, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682933735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9192024-02-20T22:42:10.388Z{"msg":"[0]R generation[0..12]: [6, 5, 1, 5, 1, 6, 6, 6, 7, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682961724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9202024-02-20T22:42:10.389Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.682988033Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9212024-02-20T22:42:10.389Z{"msg":"[1]R flush_numbers[0..12]: [10, 8, 1, 8, 1, 9, 10, 10, 11, 10, 1, 1]","v":0Wait for a query_work_queue command to finish before sending IO
9222024-02-20T22:42:10.389Z,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683020222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9232024-02-20T22:42:10.389Z{"msg":"[1]R generation[0..12]: [6, 5, 1, 5, 1, 6, 6, 6, 7, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683046631Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9242024-02-20T22:42:10.389Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.6830724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9252024-02-20T22:42:10.389Z{"msg":"[2]R flush_numbers[0..12]: [10, 8, 1, 8, 1, 9, 10, 10, 11, 10, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683097739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9262024-02-20T22:42:10.390Z{"msg":"[2]R generation[0..12]: [6, 5, 1, 5, 1, 6, 6, 6, 7, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683127048Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9272024-02-20T22:42:10.390Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683152767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9282024-02-20T22:42:10.390Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683178886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9292024-02-20T22:42:10.390Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683202885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9302024-02-20T22:42:10.390Z{"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683228744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9312024-02-20T22:42:10.391Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683251923Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9322024-02-20T22:42:10.391Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683279442Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9332024-02-20T22:42:10.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683300772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9342024-02-20T22:42:10.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683328201Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9352024-02-20T22:42:10.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.68335224Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9362024-02-20T22:42:10.392Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683378799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9372024-02-20T22:42:10.392Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683400778Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9382024-02-20T22:42:10.392Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9392024-02-20T22:42:10.392Zd0ecb075-3dd5-4e4a-8fcf-03c681ed830a is now active with session: f2bc9059-9d6d-446f-a691-ac6458b15d40","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683431737Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9402024-02-20T22:42:10.392Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683455606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167}
9412024-02-20T22:42:10.393Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:08.683484175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167,"session_id":"f2bc9059-9d6d-446f-a691-ac6458b15d40"}
9422024-02-20T22:42:10.393ZLoading write count information from file "/tmp/test_repair_verify.data"
9432024-02-20T22:42:10.393ZRun Repair workload
9442024-02-20T22:42:10.393Z01/30 Write block 526 len 16384 data: 3 3 3 3
9452024-02-20T22:42:10.393Z02/30 Write block 111 len 28672 data: 2 3 3 3 3 3 3
9462024-02-20T22:42:10.394Z03/30 Read block 437 len 8192
9472024-02-20T22:42:10.394Z04/30 Write block 14 len 40960 data: 2 2 3 3 2 2 2 2 2 2
9482024-02-20T22:42:10.394Z05/30 Write block 113 len 20480 data: 4 4 4 4 4
9492024-02-20T22:42:10.394Z06/30 Read block 189 len 40960
9502024-02-20T22:42:10.394Z07/30 Write block 230 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9512024-02-20T22:42:10.395Z08/30 Read block 379 len 16384
9522024-02-20T22:42:10.395Z09/30 Write block 382 len 20480 data: 2 2 2 2 2
9532024-02-20T22:42:10.395Z10/30 Write block 2 len 20480 data: 2 2 2 2 2
9542024-02-20T22:42:10.395Z11/30 Read block 282 len 4096
9552024-02-20T22:42:10.395Z12/30 Flush
9562024-02-20T22:42:10.396Z13/30 Flush
9572024-02-20T22:42:10.396Z14/30 Write block 522 len 4096 data: 2
9582024-02-20T22:42:10.396Z15/30 Write block 528 len 12288 data: 4 4 3
9592024-02-20T22:42:10.396Z16/30 Write block 99 len 12288 data: 2 2 2
9602024-02-20T22:42:10.396Z17/30 Write block 127 len 12288 data: 2 2 2
9612024-02-20T22:42:10.397Z18/30 Flush
9622024-02-20T22:42:10.397Z19/30 Write block 585 len 32768 data: 2 2 2 2 2 2 2 2
9632024-02-20T22:42:10.397Z20/30 Write block 180 len 12288 data: 3 3 3
9642024-02-20T22:42:10.397Z21/30 Write block 165 len 36864 data: 3 3 3 3 3 3 2 2 2
9652024-02-20T22:42:10.397Z22/30 Read block 511 len 28672
9662024-02-20T22:42:10.398Z23/30 Write block 563 len 8192 data: 2 2
9672024-02-20T22:42:10.398Z24/30 Read block 529 len 32768
9682024-02-20T22:42:10.398Z25/30 Write block 397 len 28672 data: 3 3 3 3 3 3 3
9692024-02-20T22:42:10.398Z26/30 Write block 99 len 8192 data: 3 3
9702024-02-20T22:42:10.399Z27/30 Read block 395 len 4096
9712024-02-20T22:42:10.399Z28/30 Read block 529 len 12288
9722024-02-20T22:42:10.399Z29/30 Write block 402 len 32768 data: 4 4 3 3 3 3 2 2
9732024-02-20T22:42:10.399Z30/30 Write block 465 len 20480 data: 4 4 4 4 4
9742024-02-20T22:42:10.399Z----------------------------------------------------------------
9752024-02-20T22:42:10.399Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9762024-02-20T22:42:10.400ZStates: Active Active Active
9772024-02-20T22:42:10.400ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9782024-02-20T22:42:10.400Z 1 Acked 1000 Write 4 Done Sent Done false
9792024-02-20T22:42:10.400Z 2 Acked 1001 Write 7 Done Sent Done false
9802024-02-20T22:42:10.426Z 3 Acked 1002 Read 2 Done Sent Done false
9812024-02-20T22:42:10.427Z 4 Acked 1003 Write 10 Done Sent Done false
9822024-02-20T22:42:10.427Z 5 Acked 1004 Write 5 Done Sent Done false
9832024-02-20T22:42:10.427Z 6 Acked 1005 Read 10 Done Sent Done false
9842024-02-20T22:42:10.427Z 7 Acked 1006 Write 10 Done Sent Done false
9852024-02-20T22:42:10.428Z 8 Acked 1007 Read 4 Done Sent Done false
9862024-02-20T22:42:10.428Z 9 Acked 1008 Write 5 Done Sent Done false
9872024-02-20T22:42:10.428Z 10 Acked 1009 Write 5 Done Sent Done false
9882024-02-20T22:42:10.428Z 11 Acked 1010 Read 1 Done Sent Done false
9892024-02-20T22:42:10.428Z 12 Acked 1011 Flush 0 Done Sent Done false
9902024-02-20T22:42:10.429Z 13 Acked 1012 Flush 0 Done Sent Done false
9912024-02-20T22:42:10.429Z 14 Acked 1013 Write 1 Done Sent Done false
9922024-02-20T22:42:10.429Z 15 Acked 1014 Write 3 Done Sent Done false
9932024-02-20T22:42:10.429Z 16 Acked 1015 Write 3 Done Sent Done false
9942024-02-20T22:42:10.430Z 17 Acked 1016 Write 3 Done Sent Done false
9952024-02-20T22:42:10.430Z 18 Acked 1017 Flush 0 Done Sent Done false
9962024-02-20T22:42:10.430Z 19 Acked 1018 Write 8 Done Sent Done false
9972024-02-20T22:42:10.430Z 20 Acked 1019 Write 3 Done Sent Done false
9982024-02-20T22:42:10.431Z 21 Acked 1020 Write 9 Done Sent Done false
9992024-02-20T22:42:10.431Z 22 Acked 1021 Read 7 Done Sent Done false
10002024-02-20T22:42:10.431Z 23 Acked 1022 Write 2 Done Sent Done false
10012024-02-20T22:42:10.431Z 24 Acked 1023 Read 8 Done Sent Done false
10022024-02-20T22:42:10.431Z 25 Acked 1024 Write 7 Done Sent Done false
10032024-02-20T22:42:10.431Z 26 Acked 1025 Write 2 Done Sent Done false
10042024-02-20T22:42:10.432Z 27 Acked 1026 Read 1 Done Sent Done false
10052024-02-20T22:42:10.432Z 28 Acked 1027 Read 3 Done Sent Done false
10062024-02-20T22:42:10.433Z 29 Acked 1028 Write 8 Sent Sent Sent false
10072024-02-20T22:42:10.433Z 30 Acked 1029 Write 5 Sent Sent Sent false
10082024-02-20T22:42:10.433Z STATES DS:0 DS:1 DS:2 TOTAL
10092024-02-20T22:42:10.433Z New 0 0 0 0
10102024-02-20T22:42:10.433Z Sent 2 30 2 34
10112024-02-20T22:42:10.434Z Done 28 0 28 56
10122024-02-20T22:42:10.434Z Skipped 0 0 0 0
10132024-02-20T22:42:10.434Z Error 0 0 0 0
10142024-02-20T22:42:10.434ZLast Flush: 1017 0 1017
10152024-02-20T22:42:10.434ZDownstairs last five completed:
10162024-02-20T22:42:10.435ZUpstairs last five completed: 30 29 28 27 26
10172024-02-20T22:42:10.435Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:08.735456035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1167}
10182024-02-20T22:42:10.435ZWrote out file "/tmp/test_repair_verify.data"
10192024-02-20T22:42:10.436Z
10202024-02-20T22:42:10.436ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10212024-02-20T22:42:10.436ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10222024-02-20T22:42:10.436Z 0 000-019 8 6 8 12 10 12 F F F <---
10232024-02-20T22:42:10.436Z 1 020-039 8 5 8 12 8 12 F F F <---
10242024-02-20T22:42:10.437Z 2 040-059 1 1 1 1 1 1 F F F
10252024-02-20T22:42:10.437Z 3 060-079 5 5 5 8 8 8 F F F
10262024-02-20T22:42:10.437Z 4 080-099 8 1 8 14 1 14 T F T <---
10272024-02-20T22:42:10.437Z 5 100-119 8 6 8 14 9 14 T F T <---
10282024-02-20T22:42:10.438Z 6 120-139 8 6 8 14 10 14 F F F <---
10292024-02-20T22:42:10.438Z 7 140-159 6 6 6 10 10 10 F F F
10302024-02-20T22:42:10.438Z 8 160-179 7 7 7 11 11 11 T F T <---
10312024-02-20T22:42:10.438Z 9 180-199 6 6 6 10 10 10 T F T <---
10322024-02-20T22:42:10.439Z 10 200-219 1 1 1 1 1 1 F F F
10332024-02-20T22:42:10.439Z 11 220-239 8 1 8 12 1 12 F F F <---
10342024-02-20T22:42:10.439Z 12 240-259 2 2 2 2 2 2 F F F
10352024-02-20T22:42:10.439Z 13 260-279 4 4 4 7 7 7 F F F
10362024-02-20T22:42:10.440Z 14 280-299 4 4 4 5 5 5 F F F
10372024-02-20T22:42:10.440Z 15 300-319 6 6 6 9 9 9 F F F
10382024-02-20T22:42:10.440Z 16 320-339 6 6 6 9 9 9 F F F
10392024-02-20T22:42:10.440Z 17 340-359 6 6 6 9 9 9 F F F
10402024-02-20T22:42:10.440Z 18 360-379 6 6 6 9 9 9 F F F
10412024-02-20T22:42:10.441Z 19 380-399 8 6 8 12 9 12 T F T <---
10422024-02-20T22:42:10.441Z 20 400-419 6 6 6 9 9 9 T F T <---
10432024-02-20T22:42:10.441Z 21 420-439 1 1 1 1 1 1 F F F
10442024-02-20T22:42:10.441Z 22 440-459 2 2 2 2 2 2 F F F
10452024-02-20T22:42:10.442Z 23 460-479 4 4 4 6 6 6 T F T <---
10462024-02-20T22:42:10.442Z 24 480-499 1 1 1 1 1 1 F F F
10472024-02-20T22:42:10.442Z 25 500-519 1 1 1 1 1 1 F F F
10482024-02-20T22:42:10.442Z 26 520-539 8 6 8 14 9 14 F F F <---
10492024-02-20T22:42:10.443Z 27 540-559 6 6 6 9 9 9 F F F
10502024-02-20T22:42:10.443Z 28 560-579 3 3 3 3 3 3 T F T <---
10512024-02-20T22:42:10.443Z 29 580-599 3 3 3 3 3 3 T F T <---
10522024-02-20T22:42:10.443ZMax gen: 8, Max flush: 14
10532024-02-20T22:42:10.444ZError: Difference in extent metadata found!
10542024-02-20T22:42:10.444ZA Difference in extent metadata is expected here
10552024-02-20T22:42:10.444ZOn loop 4
10562024-02-20T22:42:10.444Z
10572024-02-20T22:42:10.445ZVerifying data now
10582024-02-20T22:42:14.891ZNo change after verify
10592024-02-20T22:42:14.891ZLoop: 4 Downstairs dump after verify (and repair):
10602024-02-20T22:42:14.908ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10612024-02-20T22:42:14.908Z 0 000-019 8 8 8 12 12 12 F F F
10622024-02-20T22:42:14.908Z 1 020-039 8 8 8 12 12 12 F F F
10632024-02-20T22:42:14.908Z 2 040-059 1 1 1 1 1 1 F F F
10642024-02-20T22:42:14.908Z 3 060-079 5 5 5 8 8 8 F F F
10652024-02-20T22:42:14.908Z 4 080-099 9 9 9 15 15 15 F F F
10662024-02-20T22:42:14.908Z 5 100-119 9 9 9 15 15 15 F F F
10672024-02-20T22:42:14.908Z 6 120-139 8 8 8 14 14 14 F F F
10682024-02-20T22:42:14.909Z 7 140-159 6 6 6 10 10 10 F F F
10692024-02-20T22:42:14.909Z 8 160-179 9 9 9 15 15 15 F F F
10702024-02-20T22:42:14.909Z 9 180-199 9 9 9 15 15 15 F F F
10712024-02-20T22:42:14.909Z 10 200-219 1 1 1 1 1 1 F F F
10722024-02-20T22:42:14.909Z 11 220-239 8 8 8 12 12 12 F F F
10732024-02-20T22:42:14.909Z 12 240-259 2 2 2 2 2 2 F F F
10742024-02-20T22:42:14.909Z 13 260-279 4 4 4 7 7 7 F F F
10752024-02-20T22:42:14.909Z 14 280-299 4 4 4 5 5 5 F F F
10762024-02-20T22:42:14.909Z 15 300-319 6 6 6 9 9 9 F F F
10772024-02-20T22:42:14.909Z 16 320-339 6 6 6 9 9 9 F F F
10782024-02-20T22:42:14.909Z 17 340-359 6 6 6 9 9 9 F F F
10792024-02-20T22:42:14.910Z 18 360-379 6 6 6 9 9 9 F F F
10802024-02-20T22:42:14.910Z 19 380-399 9 9 9 15 15 15 F F F
10812024-02-20T22:42:14.910Z 20 400-419 9 9 9 15 15 15 F F F
10822024-02-20T22:42:14.910Z 21 420-439 1 1 1 1 1 1 F F F
10832024-02-20T22:42:14.910Z 22 440-459 2 2 2 2 2 2 F F F
10842024-02-20T22:42:14.910Z 23 460-479 9 9 9 15 15 15 F F F
10852024-02-20T22:42:14.910Z 24 480-499 1 1 1 1 1 1 F F F
10862024-02-20T22:42:14.910Z 25 500-519 1 1 1 1 1 1 F F F
10872024-02-20T22:42:14.910Z 26 520-539 8 8 8 14 14 14 F F F
10882024-02-20T22:42:14.910Z 27 540-559 6 6 6 9 9 9 F F F
10892024-02-20T22:42:14.910Z 28 560-579 9 9 9 15 15 15 F F F
10902024-02-20T22:42:14.911Z 29 580-599 9 9 9 15 15 15 F F F
10912024-02-20T22:42:14.911ZMax gen: 9, Max flush: 15
10922024-02-20T22:42:14.911Z
10932024-02-20T22:42:14.911ZBegin loop 5
10942024-02-20T22:42:14.911ZDownstairs to restart: 1
10952024-02-20T22:42:14.935Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458073914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175}
10962024-02-20T22:42:14.936Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458451621Z","Crucible runtime is spawned
10972024-02-20T22:42:14.936Zhostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175}
10982024-02-20T22:42:14.936Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458503569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175}
10992024-02-20T22:42:14.936Z{"msg":"Crucible 585e057e-1e1b-40ad-8075-caffd1834c04 has session id: 859059a8-fa2d-4e1c-a919-3a57786bb6e6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458532918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11002024-02-20T22:42:14.936Z{"msg":"Upstairs opts: Upstairs UUID: 585e057e-1e1b-40ad-8075-caffd1834c04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458588436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11012024-02-20T22:42:14.936Z{"msg":"Crucible stats registered with UUID: 585e057e-1e1b-40ad-8075-caffd1834c04","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458641294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11022024-02-20T22:42:14.936Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458670933Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175}
11032024-02-20T22:42:14.936Z{"msg":"585e057e-1e1b-40ad-8075-caffd1834c04 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458711492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11042024-02-20T22:42:14.936Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.45874205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11052024-02-20T22:42:14.936Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458787839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11062024-02-20T22:42:14.936Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458820677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11072024-02-20T22:42:14.936Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458876815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11082024-02-20T22:42:14.937Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458931873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11092024-02-20T22:42:14.937Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.458964542Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11102024-02-20T22:42:14.937Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459007321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11112024-02-20T22:42:14.937Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459175315Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11122024-02-20T22:42:14.937Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459214794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11132024-02-20T22:42:14.937Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459230913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11142024-02-20T22:42:14.937Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459245502Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11152024-02-20T22:42:14.937Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459259622Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11162024-02-20T22:42:14.938Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459754544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11172024-02-20T22:42:14.938Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459791743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11182024-02-20T22:42:14.938Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459808272Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11192024-02-20T22:42:14.938Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459822012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11202024-02-20T22:42:14.938Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.459856431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11212024-02-20T22:42:14.939Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.45987626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11222024-02-20T22:42:14.939Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.460387961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11232024-02-20T22:42:14.939Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.46042626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11242024-02-20T22:42:14.939Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.460450519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11252024-02-20T22:42:14.939Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.460893803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11262024-02-20T22:42:14.939Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.460923032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11272024-02-20T22:42:14.939Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.460938652Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11282024-02-20T22:42:14.939Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461013769Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11292024-02-20T22:42:14.939Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461055857Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11302024-02-20T22:42:14.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461073777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11312024-02-20T22:42:14.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461094136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11322024-02-20T22:42:14.940Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461108396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11332024-02-20T22:42:14.940Z{"msg":"[0]R flush_numbers[0..12]: [12, 12, 1, 8, 15, 15, 14, 10, 15, 15, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461129265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11342024-02-20T22:42:14.940Z{"msg":"[0]R generation[0..12]: [8, 8, 1, 5, 9, 9, 8, 6, 9, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461143694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11352024-02-20T22:42:14.940Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461164894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11362024-02-20T22:42:14.940Z{"msg":"[1]R flush_numbers[0..12]: [12, 12, 1, 8, 15, 15, 14, 10, 15, 15, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461179393Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11372024-02-20T22:42:14.941Z{"msg":"[1]R generation[0..12]: [8, 8, 1, 5, 9, 9, 8, 6, 9, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461200412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11382024-02-20T22:42:14.941Z{"msg":"Wait for a query_work_queue command to finish before sending IO
11392024-02-20T22:42:14.941Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461221892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11402024-02-20T22:42:14.941Z{"msg":"[2]R flush_numbers[0..12]: [12, 12, 1, 8, 15, 15, 14, 10, 15, 15, 1, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.46125607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11412024-02-20T22:42:14.941Z{"msg":"[2]R generation[0..12]: [8, 8, 1, 5, 9, 9, 8, 6, 9, 9, 1, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461286109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11422024-02-20T22:42:14.941Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461319178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11432024-02-20T22:42:14.942Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461334547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11442024-02-20T22:42:14.942Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461355117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11452024-02-20T22:42:14.942Z{"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461368406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11462024-02-20T22:42:14.942Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461389076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11472024-02-20T22:42:14.942Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461402975Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11482024-02-20T22:42:14.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461415075Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11492024-02-20T22:42:14.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461434874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11502024-02-20T22:42:14.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461448194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11512024-02-20T22:42:14.942Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461467863Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11522024-02-20T22:42:14.942Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461480702Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11532024-02-20T22:42:14.942Z{"msg":"585e057e-1e1b-40ad-8075-caffd1834c04 is now active with session: 859059a8-fa2d-4e1c-a919-3a57786bb6e6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461494222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11542024-02-20T22:42:14.942Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461506412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175,"session_id":"859059a8-fa2d-4e1c-a919-3a57786bb6e6"}
11552024-02-20T22:42:14.942Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:14.461525681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175}
11562024-02-20T22:42:14.943ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11572024-02-20T22:42:14.943ZLoading write count information from file "/tmp/test_repair_verify.data"
11582024-02-20T22:42:14.943ZRun Repair workload
11592024-02-20T22:42:14.943Z01/30 Flush
11602024-02-20T22:42:14.943Z02/30 Write block 185 len 24576 data: 2 2 2 2 2 2
11612024-02-20T22:42:14.943Z03/30 Flush
11622024-02-20T22:42:14.953Z04/30 Flush
11632024-02-20T22:42:14.953Z05/30 Write block 193 len 20480 data: 2 2 2 2 2
11642024-02-20T22:42:14.953Z06/30 Flush
11652024-02-20T22:42:14.953Z07/30 Write block 549 len 8192 data: 2 2
11662024-02-20T22:42:14.953Z08/30 Read block 219 len 16384
11672024-02-20T22:42:14.953Z09/30 Write block 47 len 8192 data: 2 2
11682024-02-20T22:42:14.953Z10/30 Read block 104 len 32768
11692024-02-20T22:42:14.953Z11/30 Read block 242 len 20480
11702024-02-20T22:42:14.953Z12/30 Write block 388 len 12288 data: 4 4 4
11712024-02-20T22:42:14.953Z13/30 Write block 81 len 16384 data: 2 2 2 2
11722024-02-20T22:42:14.953Z14/30 Read block 60 len 4096
11732024-02-20T22:42:14.954Z15/30 Read block 200 len 16384
11742024-02-20T22:42:14.954Z16/30 Read block 126 len 40960
11752024-02-20T22:42:14.954Z17/30 Write block 480 len 20480 data: 2 2 2 2 2
11762024-02-20T22:42:14.954Z18/30 Write block 527 len 20480 data: 4 5 5 4 3
11772024-02-20T22:42:14.954Z19/30 Flush
11782024-02-20T22:42:14.954Z20/30 Flush
11792024-02-20T22:42:14.954Z21/30 Write block 222 len 36864 data: 2 2 2 2 2 2 2 2 3
11802024-02-20T22:42:14.954Z22/30 Read block 405 len 40960
11812024-02-20T22:42:14.954Z23/30 Read block 490 len 16384
11822024-02-20T22:42:14.954Z24/30 Write block 522 len 24576 data: 3 2 2 2 4 5
11832024-02-20T22:42:14.954Z25/30 Write block 495 len 28672 data: 2 2 2 2 2 2 2
11842024-02-20T22:42:14.954Z26/30 Read block 550 len 40960
11852024-02-20T22:42:14.954Z27/30 Read block 112 len 4096
11862024-02-20T22:42:14.954Z28/30 Write block 566 len 24576 data: 2 2 2 2 2 2
11872024-02-20T22:42:14.954Z29/30 Read block 300 len 32768
11882024-02-20T22:42:14.954Z30/30 Read block 473 len 16384
11892024-02-20T22:42:14.954Z----------------------------------------------------------------
11902024-02-20T22:42:14.954Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11912024-02-20T22:42:14.954ZStates: Active Active Active
11922024-02-20T22:42:14.954ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11932024-02-20T22:42:14.954Z 1 Acked 1000 Flush 0 Done Sent Done false
11942024-02-20T22:42:14.954Z 2 Acked 1001 Write 6 Done Sent Done false
11952024-02-20T22:42:14.954Z 3 Acked 1002 Flush 0 Done Sent Done false
11962024-02-20T22:42:14.954Z 4 Acked 1003 Flush 0 Done Sent Done false
11972024-02-20T22:42:14.954Z 5 Acked 1004 Write 5 Done Sent Done false
11982024-02-20T22:42:14.955Z 6 Acked 1005 Flush 0 Done Sent Done false
11992024-02-20T22:42:14.955Z 7 Acked 1006 Write 2 Done Sent Done false
12002024-02-20T22:42:14.955Z 8 Acked 1007 Read 4 Done Sent Done false
12012024-02-20T22:42:14.955Z 9 Acked 1008 Write 2 Done Sent Done false
12022024-02-20T22:42:14.955Z 10 Acked 1009 Read 8 Done Sent Done false
12032024-02-20T22:42:14.955Z 11 Acked 1010 Read 5 Done Sent Done false
12042024-02-20T22:42:14.955Z 12 Acked 1011 Write 3 Done Sent Done false
12052024-02-20T22:42:14.955Z 13 Acked 1012 Write 4 Done Sent Done false
12062024-02-20T22:42:14.955Z 14 Acked 1013 Read 1 Done Sent Done false
12072024-02-20T22:42:14.955Z 15 Acked 1014 Read 4 Done Sent Done false
12082024-02-20T22:42:14.955Z 16 Acked 1015 Read 10 Done Sent Done false
12092024-02-20T22:42:14.955Z 17 Acked 1016 Write 5 Done Sent Done false
12102024-02-20T22:42:14.955Z 18 Acked 1017 Write 5 Done Sent Done false
12112024-02-20T22:42:14.955Z 19 Acked 1018 Flush 0 Done Sent Done false
12122024-02-20T22:42:14.955Z 20 Acked 1019 Flush 0 Done Sent Done false
12132024-02-20T22:42:14.955Z 21 Acked 1020 Write 9 Done Sent Done false
12142024-02-20T22:42:14.955Z 22 Acked 1021 Read 10 Done Sent Done false
12152024-02-20T22:42:14.955Z 23 Acked 1022 Read 4 Done Sent Done false
12162024-02-20T22:42:14.955Z 24 Acked 1023 Write 6 Done Sent Done false
12172024-02-20T22:42:14.955Z 25 Acked 1024 Write 7 Done Sent Done false
12182024-02-20T22:42:14.955Z 26 Acked 1025 Read 10 Done Sent Done false
12192024-02-20T22:42:14.955Z 27 Acked 1026 Read 1 Done Sent Done false
12202024-02-20T22:42:14.955Z 28 Acked 1027 Write 6 Done Sent Done false
12212024-02-20T22:42:14.955Z 29 Acked 1028 Read 8 Done Sent Done false
12222024-02-20T22:42:14.955Z 30 Acked 1029 Read 4 Done Sent Done false
12232024-02-20T22:42:14.956Z STATES DS:0 DS:1 DS:2 TOTAL
12242024-02-20T22:42:14.956Z New 0 0 0 0
12252024-02-20T22:42:14.956Z Sent 0 30 0 30
12262024-02-20T22:42:14.956Z Done 30 0 30 60
12272024-02-20T22:42:14.956Z Skipped 0 0 0 0
12282024-02-20T22:42:14.956Z Error 0 0 0 0
12292024-02-20T22:42:14.956ZLast Flush: 1019 0 1019
12302024-02-20T22:42:14.956ZDownstairs last five completed:
12312024-02-20T22:42:14.956ZUpstairs last five completed: 30 29 28 27 26
12322024-02-20T22:42:14.956Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:14.490929639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1175}
12332024-02-20T22:42:14.956ZWrote out file "/tmp/test_repair_verify.data"
12342024-02-20T22:42:14.961Z
12352024-02-20T22:42:14.961ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12362024-02-20T22:42:14.972ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12372024-02-20T22:42:14.972Z 0 000-019 8 8 8 12 12 12 F F F
12382024-02-20T22:42:14.972Z 1 020-039 8 8 8 12 12 12 F F F
12392024-02-20T22:42:14.972Z 2 040-059 10 1 10 20 1 20 F F F <---
12402024-02-20T22:42:14.972Z 3 060-079 5 5 5 8 8 8 F F F
12412024-02-20T22:42:14.972Z 4 080-099 10 9 10 20 15 20 F F F <---
12422024-02-20T22:42:14.973Z 5 100-119 9 9 9 15 15 15 F F F
12432024-02-20T22:42:14.973Z 6 120-139 8 8 8 14 14 14 F F F
12442024-02-20T22:42:14.973Z 7 140-159 6 6 6 10 10 10 F F F
12452024-02-20T22:42:14.973Z 8 160-179 9 9 9 15 15 15 F F F
12462024-02-20T22:42:14.973Z 9 180-199 10 9 10 19 15 19 F F F <---
12472024-02-20T22:42:14.973Z 10 200-219 1 1 1 1 1 1 F F F
12482024-02-20T22:42:14.977Z 11 220-239 8 8 8 12 12 12 T F T <---
12492024-02-20T22:42:14.978Z 12 240-259 2 2 2 2 2 2 F F F
12502024-02-20T22:42:14.978Z 13 260-279 4 4 4 7 7 7 F F F
12512024-02-20T22:42:14.978Z 14 280-299 4 4 4 5 5 5 F F F
12522024-02-20T22:42:14.981Z 15 300-319 6 6 6 9 9 9 F F F
12532024-02-20T22:42:14.981Z 16 320-339 6 6 6 9 9 9 F F F
12542024-02-20T22:42:14.982Z 17 340-359 6 6 6 9 9 9 F F F
12552024-02-20T22:42:14.982Z 18 360-379 6 6 6 9 9 9 F F F
12562024-02-20T22:42:14.982Z 19 380-399 10 9 10 20 15 20 F F F <---
12572024-02-20T22:42:14.982Z 20 400-419 9 9 9 15 15 15 F F F
12582024-02-20T22:42:14.983Z 21 420-439 1 1 1 1 1 1 F F F
12592024-02-20T22:42:14.983Z 22 440-459 2 2 2 2 2 2 F F F
12602024-02-20T22:42:14.983Z 23 460-479 9 9 9 15 15 15 F F F
12612024-02-20T22:42:14.983Z 24 480-499 10 1 10 20 1 20 T F T <---
12622024-02-20T22:42:14.983Z 25 500-519 1 1 1 1 1 1 T F T <---
12632024-02-20T22:42:14.983Z 26 520-539 10 8 10 20 14 20 T F T <---
12642024-02-20T22:42:14.984Z 27 540-559 10 6 10 20 9 20 F F F <---
12652024-02-20T22:42:14.984Z 28 560-579 9 9 9 15 15 15 T F T <---
12662024-02-20T22:42:14.984Z 29 580-599 9 9 9 15 15 15 F F F
12672024-02-20T22:42:14.984ZMax gen: 10, Max flush: 20
12682024-02-20T22:42:14.984ZError: Difference in extent metadata found!
12692024-02-20T22:42:14.984ZA Difference in extent metadata is expected here
12702024-02-20T22:42:14.984ZOn loop 5
12712024-02-20T22:42:14.984Z
12722024-02-20T22:42:14.985ZVerifying data now
12732024-02-20T22:42:21.147ZNo change after verify
12742024-02-20T22:42:21.147ZLoop: 5 Downstairs dump after verify (and repair):
12752024-02-20T22:42:21.174ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12762024-02-20T22:42:21.174Z 0 000-019 8 8 8 12 12 12 F F F
12772024-02-20T22:42:21.175Z 1 020-039 8 8 8 12 12 12 F F F
12782024-02-20T22:42:21.175Z 2 040-059 10 10 10 20 20 20 F F F
12792024-02-20T22:42:21.175Z 3 060-079 5 5 5 8 8 8 F F F
12802024-02-20T22:42:21.175Z 4 080-099 10 10 10 20 20 20 F F F
12812024-02-20T22:42:21.175Z 5 100-119 9 9 9 15 15 15 F F F
12822024-02-20T22:42:21.175Z 6 120-139 8 8 8 14 14 14 F F F
12832024-02-20T22:42:21.175Z 7 140-159 6 6 6 10 10 10 F F F
12842024-02-20T22:42:21.175Z 8 160-179 9 9 9 15 15 15 F F F
12852024-02-20T22:42:21.176Z 9 180-199 10 10 10 19 19 19 F F F
12862024-02-20T22:42:21.176Z 10 200-219 1 1 1 1 1 1 F F F
12872024-02-20T22:42:21.176Z 11 220-239 11 11 11 21 21 21 F F F
12882024-02-20T22:42:21.176Z 12 240-259 2 2 2 2 2 2 F F F
12892024-02-20T22:42:21.176Z 13 260-279 4 4 4 7 7 7 F F F
12902024-02-20T22:42:21.176Z 14 280-299 4 4 4 5 5 5 F F F
12912024-02-20T22:42:21.176Z 15 300-319 6 6 6 9 9 9 F F F
12922024-02-20T22:42:21.176Z 16 320-339 6 6 6 9 9 9 F F F
12932024-02-20T22:42:21.176Z 17 340-359 6 6 6 9 9 9 F F F
12942024-02-20T22:42:21.176Z 18 360-379 6 6 6 9 9 9 F F F
12952024-02-20T22:42:21.176Z 19 380-399 10 10 10 20 20 20 F F F
12962024-02-20T22:42:21.177Z 20 400-419 9 9 9 15 15 15 F F F
12972024-02-20T22:42:21.177Z 21 420-439 1 1 1 1 1 1 F F F
12982024-02-20T22:42:21.177Z 22 440-459 2 2 2 2 2 2 F F F
12992024-02-20T22:42:21.177Z 23 460-479 9 9 9 15 15 15 F F F
13002024-02-20T22:42:21.177Z 24 480-499 11 11 11 21 21 21 F F F
13012024-02-20T22:42:21.177Z 25 500-519 11 11 11 21 21 21 F F F
13022024-02-20T22:42:21.177Z 26 520-539 11 11 11 21 21 21 F F F
13032024-02-20T22:42:21.177Z 27 540-559 10 10 10 20 20 20 F F F
13042024-02-20T22:42:21.177Z 28 560-579 11 11 11 21 21 21 F F F
13052024-02-20T22:42:21.177Z 29 580-599 9 9 9 15 15 15 F F F
13062024-02-20T22:42:21.178ZMax gen: 11, Max flush: 21
13072024-02-20T22:42:21.178Z
13082024-02-20T22:42:21.178ZBegin loop 6
13092024-02-20T22:42:21.178ZDownstairs to restart: 0
13102024-02-20T22:42:21.210Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709096775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183}
13112024-02-20T22:42:21.210Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709337486Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183}
13122024-02-20T22:42:21.210Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709362795Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183}
13132024-02-20T22:42:21.211Z{"msg":"Crucible 2bb2c452-0afa-45f1-8beb-86b22350fd99 has session id: 99d8322d-23b3-44ae-aa09-8f71015029dd","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709377354Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13142024-02-20T22:42:21.211Z{"msg":"Upstairs opts: Upstairs UUID: 2bb2c452-0afa-45f1-8beb-86b22350fd99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
13152024-02-20T22:42:21.211Z,"time":"2024-02-20T22:42:20.709397694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13162024-02-20T22:42:21.211Z{"msg":"Crucible stats registered with UUID: 2bb2c452-0afa-45f1-8beb-86b22350fd99","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709452992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13172024-02-20T22:42:21.212Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709486361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183}
13182024-02-20T22:42:21.212Z{"msg":"2bb2c452-0afa-45f1-8beb-86b22350fd99 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709623396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13192024-02-20T22:42:21.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709655655Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13202024-02-20T22:42:21.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709701123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13212024-02-20T22:42:21.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709741012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13222024-02-20T22:42:21.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.70977869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13232024-02-20T22:42:21.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709805129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13242024-02-20T22:42:21.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709830838Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13252024-02-20T22:42:21.214Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709858157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13262024-02-20T22:42:21.214Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709902776Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13272024-02-20T22:42:21.214Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709942824Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13282024-02-20T22:42:21.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.709969753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13292024-02-20T22:42:21.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.710017562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13302024-02-20T22:42:21.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71006145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13312024-02-20T22:42:21.215Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71062407Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13322024-02-20T22:42:21.215Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.710692637Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13332024-02-20T22:42:21.215Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.710777774Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13342024-02-20T22:42:21.215Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.710818873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13352024-02-20T22:42:21.216Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.710852092Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13362024-02-20T22:42:21.216Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.710874481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13372024-02-20T22:42:21.216Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711242518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13382024-02-20T22:42:21.216Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711279956Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13392024-02-20T22:42:21.216Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711312675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13402024-02-20T22:42:21.217Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71174485Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13412024-02-20T22:42:21.217Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711826027Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13422024-02-20T22:42:21.217Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711856416Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13432024-02-20T22:42:21.217Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711874625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13442024-02-20T22:42:21.217Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711906934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13452024-02-20T22:42:21.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711922753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13462024-02-20T22:42:21.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711944183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13472024-02-20T22:42:21.218Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711959492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13482024-02-20T22:42:21.218Z{"msg":"[0]R flush_numbers[0..12]: [12, 12, 20, 8, 20, 15, 14, 10, 15, 19, 1, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711981591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13492024-02-20T22:42:21.218Z{"msg":"[0]R generation[0..12]: [8, 8, 10, 5, 10, 9, 8, 6, 9, 10, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.711996811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13502024-02-20T22:42:21.219Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71201149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13512024-02-20T22:42:21.219Z{"msg":"[1]R flush_numbers[0..12]: [12, 12, 20, 8, 20, 15, 14, 10, 15, 19, 1, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71202591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13522024-02-20T22:42:21.219Z{"msg":"[1]R generation[0..12]: [8, 8, 10, 5, 10, 9, 8, 6, 9, 10, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712040419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13532024-02-20T22:42:21.219Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712062008Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13542024-02-20T22:42:21.220Z{"msg":"[2]R flush_numbers[0..12]: [12, 12, 20, 8, 20, 15, 14, 10, 15, 19, 1, 21]",Wait for a query_work_queue command to finish before sending IO
13552024-02-20T22:42:21.220Z"v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712081998Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13562024-02-20T22:42:21.220Z{"msg":"[2]R generation[0..12]: [8, 8, 10, 5, 10, 9, 8, 6, 9, 10, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712103147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13572024-02-20T22:42:21.220Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712117706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13582024-02-20T22:42:21.220Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712137766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13592024-02-20T22:42:21.221Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712151715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13602024-02-20T22:42:21.221Z{"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712172325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13612024-02-20T22:42:21.221Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712186084Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13622024-02-20T22:42:21.221Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712200044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13632024-02-20T22:42:21.222Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712219063Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13642024-02-20T22:42:21.222Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712232532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13652024-02-20T22:42:21.222Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712253031Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13662024-02-20T22:42:21.222Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712267521Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13672024-02-20T22:42:21.223Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71228671Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13682024-02-20T22:42:21.223Z{"msg":"2bb2c452-0afa-45f1-8beb-86b22350fd99 is now active with session: 99d8322d-23b3-44ae-aa09-8f71015029dd","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.71230007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13692024-02-20T22:42:21.223Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712318759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183,"session_id":"99d8322d-23b3-44ae-aa09-8f71015029dd"}
13702024-02-20T22:42:21.223Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:20.712331539Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183}
13712024-02-20T22:42:21.223ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13722024-02-20T22:42:21.223ZLoading write count information from file "/tmp/test_repair_verify.data"
13732024-02-20T22:42:21.223ZRun Repair workload
13742024-02-20T22:42:21.224Z01/30 Write block 479 len 40960 data: 2 3 3 3 3 3 2 2 2 2
13752024-02-20T22:42:21.224Z02/30 Write block 4 len 8192 data: 3 3
13762024-02-20T22:42:21.224Z03/30 Read block 95 len 40960
13772024-02-20T22:42:21.224Z04/30 Read block 308 len 20480
13782024-02-20T22:42:21.224Z05/30 Read block 502 len 40960
13792024-02-20T22:42:21.224Z06/30 Write block 414 len 32768 data: 3 3 3 2 2 2 2 2
13802024-02-20T22:42:21.224Z07/30 Write block 108 len 24576 data: 2 2 2 3 4 5
13812024-02-20T22:42:21.225Z08/30 Flush
13822024-02-20T22:42:21.225Z09/30 Write block 576 len 20480 data: 3 3 3 3 3
13832024-02-20T22:42:21.225Z10/30 Write block 149 len 12288 data: 3 3 3
13842024-02-20T22:42:21.225Z11/30 Read block 133 len 8192
13852024-02-20T22:42:21.236Z12/30 Write block 392 len 24576 data: 4 4 4 4 3 4
13862024-02-20T22:42:21.236Z13/30 Write block 30 len 40960 data: 3 3 3 3 3 3 2 2 2 2
13872024-02-20T22:42:21.236Z14/30 Read block 99 len 24576
13882024-02-20T22:42:21.236Z15/30 Write block 493 len 8192 data: 2 2
13892024-02-20T22:42:21.237Z16/30 Write block 406 len 24576 data: 4 4 3 3 3 3
13902024-02-20T22:42:21.237Z17/30 Flush
13912024-02-20T22:42:21.253Z18/30 Write block 462 len 40960 data: 4 4 4 5 5 5 5 5 3 2
13922024-02-20T22:42:21.253Z19/30 Read block 477 len 12288
13932024-02-20T22:42:21.253Z20/30 Write block 311 len 20480 data: 2 2 2 2 2
13942024-02-20T22:42:21.254Z21/30 Write block 229 len 28672 data: 3 4 3 3 3 3 3
13952024-02-20T22:42:21.254Z22/30 Read block 20 len 12288
13962024-02-20T22:42:21.254Z23/30 Read block 5 len 36864
13972024-02-20T22:42:21.254Z24/30 Read block 263 len 8192
13982024-02-20T22:42:21.254Z25/30 Write block 179 len 28672 data: 3 4 4 4 3 3 3
13992024-02-20T22:42:21.254Z26/30 Read block 81 len 8192
14002024-02-20T22:42:21.255Z27/30 Write block 456 len 20480 data: 3 3 3 2 2
14012024-02-20T22:42:21.276Z28/30 Write block 363 len 40960 data: 4 4 4 4 3 3 2 2 2 2
14022024-02-20T22:42:21.277Z29/30 Read block 349 len 12288
14032024-02-20T22:42:21.277Z30/30 Read block 42 len 8192
14042024-02-20T22:42:21.277Z----------------------------------------------------------------
14052024-02-20T22:42:21.277Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14062024-02-20T22:42:21.277ZStates: Active Active Active
14072024-02-20T22:42:21.277ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14082024-02-20T22:42:21.278Z 1 Acked 1000 Write 10 Sent Done Done false
14092024-02-20T22:42:21.278Z 2 Acked 1001 Write 2 Sent Done Done false
14102024-02-20T22:42:21.278Z 3 Acked 1002 Read 10 Sent Done Done false
14112024-02-20T22:42:21.278Z 4 Acked 1003 Read 5 Sent Done Done false
14122024-02-20T22:42:21.278Z 5 Acked 1004 Read 10 Sent Done Done false
14132024-02-20T22:42:21.279Z 6 Acked 1005 Write 8 Sent Done Done false
14142024-02-20T22:42:21.279Z 7 Acked 1006 Write 6 Sent Done Done false
14152024-02-20T22:42:21.279Z 8 Acked 1007 Flush 0 Sent Done Done false
14162024-02-20T22:42:21.279Z 9 Acked 1008 Write 5 Sent Done Done false
14172024-02-20T22:42:21.279Z 10 Acked 1009 Write 3 Sent Done Done false
14182024-02-20T22:42:21.279Z 11 Acked 1010 Read 2 Sent Done Done false
14192024-02-20T22:42:21.280Z 12 Acked 1011 Write 6 Sent Done Done false
14202024-02-20T22:42:21.280Z 13 Acked 1012 Write 10 Sent Done Done false
14212024-02-20T22:42:21.280Z 14 Acked 1013 Read 6 Sent Done Done false
14222024-02-20T22:42:21.280Z 15 Acked 1014 Write 2 Sent Done Done false
14232024-02-20T22:42:21.280Z 16 Acked 1015 Write 6 Sent Done Done false
14242024-02-20T22:42:21.281Z 17 Acked 1016 Flush 0 Sent Done Done false
14252024-02-20T22:42:21.281Z 18 Acked 1017 Write 10 Sent Done Done false
14262024-02-20T22:42:21.281Z 19 Acked 1018 Read 3 Sent Done Done false
14272024-02-20T22:42:21.281Z 20 Acked 1019 Write 5 Sent Done Done false
14282024-02-20T22:42:21.281Z 21 Acked 1020 Write 7 Sent Done Done false
14292024-02-20T22:42:21.281Z 22 Acked 1021 Read 3 Sent Done Done false
14302024-02-20T22:42:21.281Z 23 Acked 1022 Read 9 Sent Done Done false
14312024-02-20T22:42:21.282Z 24 Acked 1023 Read 2 Sent Done Done false
14322024-02-20T22:42:21.282Z 25 Acked 1024 Write 7 Sent Done Done false
14332024-02-20T22:42:21.282Z 26 Acked 1025 Read 2 Sent Done Done false
14342024-02-20T22:42:21.282Z 27 Acked 1026 Write 5 Sent Done Done false
14352024-02-20T22:42:21.282Z 28 Acked 1027 Write 10 Sent Done Done false
14362024-02-20T22:42:21.282Z 29 Acked 1028 Read 3 Sent Done Done false
14372024-02-20T22:42:21.282Z 30 Acked 1029 Read 2 Sent Done Done false
14382024-02-20T22:42:21.283Z STATES DS:0 DS:1 DS:2 TOTAL
14392024-02-20T22:42:21.283Z New 0 0 0 0
14402024-02-20T22:42:21.283Z Sent 30 0 0 30
14412024-02-20T22:42:21.283Z Done 0 30 30 60
14422024-02-20T22:42:21.283Z Skipped 0 0 0 0
14432024-02-20T22:42:21.283Z Error 0 0 0 0
14442024-02-20T22:42:21.283ZLast Flush: 0 1016 1016
14452024-02-20T22:42:21.284ZDownstairs last five completed:
14462024-02-20T22:42:21.284ZUpstairs last five completed: 30 29 28 27 26
14472024-02-20T22:42:21.284Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:20.792817959Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1183}
14482024-02-20T22:42:21.284ZWrote out file "/tmp/test_repair_verify.data"
14492024-02-20T22:42:21.284Z
14502024-02-20T22:42:21.284ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14512024-02-20T22:42:21.834ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14522024-02-20T22:42:21.834Z 0 000-019 8 12 12 12 22 22 F F F <---
14532024-02-20T22:42:21.834Z 1 020-039 8 12 12 12 23 23 F F F <---
14542024-02-20T22:42:21.834Z 2 040-059 10 10 10 20 20 20 F F F
14552024-02-20T22:42:21.834Z 3 060-079 5 5 5 8 8 8 F F F
14562024-02-20T22:42:21.834Z 4 080-099 10 10 10 20 20 20 F F F
14572024-02-20T22:42:21.834Z 5 100-119 9 12 12 15 22 22 F F F <---
14582024-02-20T22:42:21.835Z 6 120-139 8 8 8 14 14 14 F F F
14592024-02-20T22:42:21.835Z 7 140-159 6 12 12 10 23 23 F F F <---
14602024-02-20T22:42:21.835Z 8 160-179 9 9 9 15 15 15 F T T <---
14612024-02-20T22:42:21.835Z 9 180-199 10 10 10 19 19 19 F T T <---
14622024-02-20T22:42:21.835Z 10 200-219 1 1 1 1 1 1 F F F
14632024-02-20T22:42:21.835Z 11 220-239 11 11 11 21 21 21 F T T <---
14642024-02-20T22:42:21.835Z 12 240-259 2 2 2 2 2 2 F F F
14652024-02-20T22:42:21.835Z 13 260-279 4 4 4 7 7 7 F F F
14662024-02-20T22:42:21.835Z 14 280-299 4 4 4 5 5 5 F F F
14672024-02-20T22:42:21.835Z 15 300-319 6 6 6 9 9 9 F T T <---
14682024-02-20T22:42:21.835Z 16 320-339 6 6 6 9 9 9 F F F
14692024-02-20T22:42:21.835Z 17 340-359 6 6 6 9 9 9 F F F
14702024-02-20T22:42:21.835Z 18 360-379 6 6 6 9 9 9 F T T <---
14712024-02-20T22:42:21.835Z 19 380-399 10 12 12 20 23 23 F F F <---
14722024-02-20T22:42:21.835Z 20 400-419 9 12 12 15 23 23 F F F <---
14732024-02-20T22:42:21.835Z 21 420-439 1 12 12 1 22 22 F F F <---
14742024-02-20T22:42:21.835Z 22 440-459 2 2 2 2 2 2 F T T <---
14752024-02-20T22:42:21.835Z 23 460-479 9 12 12 15 22 22 F T T <---
14762024-02-20T22:42:21.835Z 24 480-499 11 12 12 21 23 23 F F F <---
14772024-02-20T22:42:21.835Z 25 500-519 11 11 11 21 21 21 F F F
14782024-02-20T22:42:21.835Z 26 520-539 11 11 11 21 21 21 F F F
14792024-02-20T22:42:21.835Z 27 540-559 10 10 10 20 20 20 F F F
14802024-02-20T22:42:21.835Z 28 560-579 11 12 12 21 23 23 F F F <---
14812024-02-20T22:42:21.835Z 29 580-599 9 12 12 15 23 23 F F F <---
14822024-02-20T22:42:21.836ZMax gen: 12, Max flush: 23
14832024-02-20T22:42:21.836ZError: Difference in extent metadata found!
14842024-02-20T22:42:21.836ZA Difference in extent metadata is expected here
14852024-02-20T22:42:21.836ZOn loop 6
14862024-02-20T22:42:21.836Z
14872024-02-20T22:42:21.836ZVerifying data now
14882024-02-20T22:42:26.998ZNo change after verify
14892024-02-20T22:42:26.998ZLoop: 6 Downstairs dump after verify (and repair):
14902024-02-20T22:42:27.015ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14912024-02-20T22:42:27.015Z 0 000-019 12 12 12 22 22 22 F F F
14922024-02-20T22:42:27.015Z 1 020-039 12 12 12 23 23 23 F F F
14932024-02-20T22:42:27.015Z 2 040-059 10 10 10 20 20 20 F F F
14942024-02-20T22:42:27.015Z 3 060-079 5 5 5 8 8 8 F F F
14952024-02-20T22:42:27.016Z 4 080-099 10 10 10 20 20 20 F F F
14962024-02-20T22:42:27.016Z 5 100-119 12 12 12 22 22 22 F F F
14972024-02-20T22:42:27.016Z 6 120-139 8 8 8 14 14 14 F F F
14982024-02-20T22:42:27.016Z 7 140-159 12 12 12 23 23 23 F F F
14992024-02-20T22:42:27.016Z 8 160-179 13 13 13 24 24 24 F F F
15002024-02-20T22:42:27.016Z 9 180-199 13 13 13 24 24 24 F F F
15012024-02-20T22:42:27.016Z 10 200-219 1 1 1 1 1 1 F F F
15022024-02-20T22:42:27.016Z 11 220-239 13 13 13 24 24 24 F F F
15032024-02-20T22:42:27.016Z 12 240-259 2 2 2 2 2 2 F F F
15042024-02-20T22:42:27.016Z 13 260-279 4 4 4 7 7 7 F F F
15052024-02-20T22:42:27.016Z 14 280-299 4 4 4 5 5 5 F F F
15062024-02-20T22:42:27.016Z 15 300-319 13 13 13 24 24 24 F F F
15072024-02-20T22:42:27.017Z 16 320-339 6 6 6 9 9 9 F F F
15082024-02-20T22:42:27.017Z 17 340-359 6 6 6 9 9 9 F F F
15092024-02-20T22:42:27.017Z 18 360-379 13 13 13 24 24 24 F F F
15102024-02-20T22:42:27.017Z 19 380-399 12 12 12 23 23 23 F F F
15112024-02-20T22:42:27.017Z 20 400-419 12 12 12 23 23 23 F F F
15122024-02-20T22:42:27.017Z 21 420-439 12 12 12 22 22 22 F F F
15132024-02-20T22:42:27.017Z 22 440-459 13 13 13 24 24 24 F F F
15142024-02-20T22:42:27.017Z 23 460-479 13 13 13 24 24 24 F F F
15152024-02-20T22:42:27.017Z 24 480-499 12 12 12 23 23 23 F F F
15162024-02-20T22:42:27.017Z 25 500-519 11 11 11 21 21 21 F F F
15172024-02-20T22:42:27.017Z 26 520-539 11 11 11 21 21 21 F F F
15182024-02-20T22:42:27.017Z 27 540-559 10 10 10 20 20 20 F F F
15192024-02-20T22:42:27.018Z 28 560-579 12 12 12 23 23 23 F F F
15202024-02-20T22:42:27.018Z 29 580-599 12 12 12 23 23 23 F F F
15212024-02-20T22:42:27.018ZMax gen: 13, Max flush: 24
15222024-02-20T22:42:27.030Z
15232024-02-20T22:42:27.030ZBegin loop 7
15242024-02-20T22:42:27.030ZDownstairs to restart: 1
15252024-02-20T22:42:27.054Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.573779368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191}
15262024-02-20T22:42:27.054Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574088108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191}
15272024-02-20T22:42:27.055Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
15282024-02-20T22:42:27.055Z,"time":"2024-02-20T22:42:26.574138696Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191}
15292024-02-20T22:42:27.055Z{"msg":"Crucible 63f71574-6248-4d63-af90-0879d6d0d613 has session id: ea8f288e-77e0-42e7-81bb-4aba4273a912","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574171694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15302024-02-20T22:42:27.055Z{"msg":"Upstairs opts: Upstairs UUID: 63f71574-6248-4d63-af90-0879d6d0d613, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574220193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15312024-02-20T22:42:27.055Z{"msg":"Crucible stats registered with UUID: 63f71574-6248-4d63-af90-0879d6d0d613","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574285041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15322024-02-20T22:42:27.055Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574325669Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191}
15332024-02-20T22:42:27.055Z{"msg":"63f71574-6248-4d63-af90-0879d6d0d613 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574348958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15342024-02-20T22:42:27.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574388097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15352024-02-20T22:42:27.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574415296Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15362024-02-20T22:42:27.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574437225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15372024-02-20T22:42:27.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574473644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15382024-02-20T22:42:27.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574514162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15392024-02-20T22:42:27.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574557501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15402024-02-20T22:42:27.056Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.57458627Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15412024-02-20T22:42:27.056Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574620198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15422024-02-20T22:42:27.056Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574660447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15432024-02-20T22:42:27.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574697366Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15442024-02-20T22:42:27.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574741594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15452024-02-20T22:42:27.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.574767223Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15462024-02-20T22:42:27.057Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.575372202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15472024-02-20T22:42:27.057Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.57541349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15482024-02-20T22:42:27.057Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.575439839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15492024-02-20T22:42:27.057Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.575470198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15502024-02-20T22:42:27.057Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.575494977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15512024-02-20T22:42:27.057Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.575520306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15522024-02-20T22:42:27.057Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576030278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15532024-02-20T22:42:27.057Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576075266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15542024-02-20T22:42:27.057Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576163523Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15552024-02-20T22:42:27.057Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576552289Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15562024-02-20T22:42:27.057Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576598378Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15572024-02-20T22:42:27.057Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576623957Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15582024-02-20T22:42:27.058Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576663505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15592024-02-20T22:42:27.058Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576695184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15602024-02-20T22:42:27.058Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576711614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15612024-02-20T22:42:27.058Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576725363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15622024-02-20T22:42:27.058Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576746302Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15632024-02-20T22:42:27.058Z{"msg":"[0]R flush_numbers[0..12]: [22, 23, 20, 8, 20, 22, 14, 23, 24, 24, 1, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576761252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15642024-02-20T22:42:27.058Z{"msg":"[0]R generation[0..12]: [12, 12, 10, 5, 10, 12, 8, 12, 13, 13, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576782211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15652024-02-20T22:42:27.058Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.5767971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15662024-02-20T22:42:27.058Z{"msg":"[1]R flush_numbers[0..12]: [22, 23, 20, 8, 20, 22, 14, 23, 24, 24, 1, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.57681841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15672024-02-20T22:42:27.058Z{"msg":"[1]R generation[0..12]: [12, 12, 10, 5, 10, 12, 8, 12, 13, 13, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576832579Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15682024-02-20T22:42:27.058Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576860108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15692024-02-20T22:42:27.058ZWait for a query_work_queue command to finish before sending IO
15702024-02-20T22:42:27.059Z{"msg":"[2]R flush_numbers[0..12]: [22, 23, 20, 8, 20, 22, 14, 23, 24, 24, 1, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576901597Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15712024-02-20T22:42:27.059Z{"msg":"[2]R generation[0..12]: [12, 12, 10, 5, 10, 12, 8, 12, 13, 13, 1, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576935846Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15722024-02-20T22:42:27.059Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.576974234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15732024-02-20T22:42:27.059Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577010803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15742024-02-20T22:42:27.059Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577045282Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15752024-02-20T22:42:27.059Z{"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.57708037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15762024-02-20T22:42:27.059Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577119269Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15772024-02-20T22:42:27.059Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577150198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15782024-02-20T22:42:27.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577167977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15792024-02-20T22:42:27.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577202826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15802024-02-20T22:42:27.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577236005Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15812024-02-20T22:42:27.059Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577256534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15822024-02-20T22:42:27.059Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577280113Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15832024-02-20T22:42:27.059Z{"msg":"63f71574-6248-4d63-af90-0879d6d0d613 is now active with session: ea8f288e-77e0-42e7-81bb-4aba4273a912","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577312252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15842024-02-20T22:42:27.059Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.577342741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191,"session_id":"ea8f288e-77e0-42e7-81bb-4aba4273a912"}
15852024-02-20T22:42:27.060Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15862024-02-20T22:42:27.060ZThe guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:26.57736903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191}
15872024-02-20T22:42:27.060ZLoading write count information from file "/tmp/test_repair_verify.data"
15882024-02-20T22:42:27.060ZRun Repair workload
15892024-02-20T22:42:27.060Z01/30 Flush
15902024-02-20T22:42:27.060Z02/30 Flush
15912024-02-20T22:42:27.084Z03/30 Write block 582 len 36864 data: 3 3 2 3 3 3 3 3 3
15922024-02-20T22:42:27.085Z04/30 Read block 126 len 28672
15932024-02-20T22:42:27.085Z05/30 Read block 362 len 24576
15942024-02-20T22:42:27.085Z06/30 Read block 510 len 8192
15952024-02-20T22:42:27.085Z07/30 Read block 24 len 16384
15962024-02-20T22:42:27.085Z08/30 Write block 585 len 12288 data: 4 4 4
15972024-02-20T22:42:27.085Z09/30 Write block 439 len 16384 data: 2 2 2 2
15982024-02-20T22:42:27.086Z10/30 Read block 173 len 4096
15992024-02-20T22:42:27.086Z11/30 Write block 199 len 4096 data: 2
16002024-02-20T22:42:27.086Z12/30 Read block 6 len 28672
16012024-02-20T22:42:27.086Z13/30 Write block 547 len 8192 data: 2 2
16022024-02-20T22:42:27.086Z14/30 Read block 424 len 12288
16032024-02-20T22:42:27.086Z15/30 Read block 569 len 40960
16042024-02-20T22:42:27.086Z16/30 Write block 360 len 24576 data: 3 3 3 5 5 5
16052024-02-20T22:42:27.087Z17/30 Read block 474 len 32768
16062024-02-20T22:42:27.087Z18/30 Read block 191 len 32768
16072024-02-20T22:42:27.087Z19/30 Read block 180 len 36864
16082024-02-20T22:42:27.087Z20/30 Read block 400 len 32768
16092024-02-20T22:42:27.087Z21/30 Flush
16102024-02-20T22:42:27.088Z22/30 Flush
16112024-02-20T22:42:27.088Z23/30 Write block 219 len 20480 data: 2 2 2 3 3
16122024-02-20T22:42:27.088Z24/30 Write block 109 len 20480 data: 3 3 4 5 6
16132024-02-20T22:42:27.088Z25/30 Flush
16142024-02-20T22:42:27.088Z26/30 Write block 195 len 32768 data: 3 3 3 2 3 2 2 2
16152024-02-20T22:42:27.088Z27/30 Read block 94 len 28672
16162024-02-20T22:42:27.088Z28/30 Read block 394 len 32768
16172024-02-20T22:42:27.089Z29/30 Write block 372 len 28672 data: 3 2 2 2 2 2 2
16182024-02-20T22:42:27.090Z30/30 Write block 138 len 28672 data: 3 2 2 2 2 2 2
16192024-02-20T22:42:27.090Z----------------------------------------------------------------
16202024-02-20T22:42:27.090Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:28
16212024-02-20T22:42:27.090ZStates: Active Active Active
16222024-02-20T22:42:27.091ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16232024-02-20T22:42:27.091Z 3 Acked 1002 Write 9 Done Sent Done false
16242024-02-20T22:42:27.091Z 4 Acked 1003 Read 7 Done Sent Done false
16252024-02-20T22:42:27.091Z 5 Acked 1004 Read 6 Done Sent Done false
16262024-02-20T22:42:27.092Z 6 Acked 1005 Read 2 Done Sent Done false
16272024-02-20T22:42:27.092Z 7 Acked 1006 Read 4 Done Sent Done false
16282024-02-20T22:42:27.092Z 8 Acked 1007 Write 3 Done Sent Done false
16292024-02-20T22:42:27.092Z 9 Acked 1008 Write 4 Done Sent Done false
16302024-02-20T22:42:27.092Z 10 Acked 1009 Read 1 Done Sent Done false
16312024-02-20T22:42:27.093Z 11 Acked 1010 Write 1 Done Sent Done false
16322024-02-20T22:42:27.105Z 12 Acked 1011 Read 7 Done Sent Done false
16332024-02-20T22:42:27.105Z 13 Acked 1012 Write 2 Done Sent Done false
16342024-02-20T22:42:27.105Z 14 Acked 1013 Read 3 Done Sent Done false
16352024-02-20T22:42:27.105Z 15 Acked 1014 Read 10 Done Sent Done false
16362024-02-20T22:42:27.106Z 16 Acked 1015 Write 6 Done Sent Done false
16372024-02-20T22:42:27.106Z 17 Acked 1016 Read 8 Done Sent Done false
16382024-02-20T22:42:27.106Z 18 Acked 1017 Read 8 Done Sent Done false
16392024-02-20T22:42:27.106Z 19 Acked 1018 Read 9 Done Sent Done false
16402024-02-20T22:42:27.106Z 20 Acked 1019 Read 8 Done Sent Done false
16412024-02-20T22:42:27.107Z 21 Acked 1020 Flush 0 Done Sent Done false
16422024-02-20T22:42:27.107Z 22 Acked 1021 Flush 0 Done Sent Done false
16432024-02-20T22:42:27.107Z 23 Acked 1022 Write 5 Done Sent Done false
16442024-02-20T22:42:27.107Z 24 Acked 1023 Write 5 Done Sent Done false
16452024-02-20T22:42:27.107Z 25 Acked 1024 Flush 0 Done Sent Done false
16462024-02-20T22:42:27.107Z 26 Acked 1025 Write 8 Done Sent Done false
16472024-02-20T22:42:27.108Z 27 Acked 1026 Read 7 Done Sent Done false
16482024-02-20T22:42:27.108Z 28 Acked 1027 Read 8 Done Sent Done false
16492024-02-20T22:42:27.108Z 29 Acked 1028 Write 7 Done Sent Done false
16502024-02-20T22:42:27.108Z 30 Acked 1029 Write 7 Sent Sent Sent false
16512024-02-20T22:42:27.108Z STATES DS:0 DS:1 DS:2 TOTAL
16522024-02-20T22:42:27.109Z New 0 0 0 0
16532024-02-20T22:42:27.109Z Sent 1 28 1 30
16542024-02-20T22:42:27.109Z Done 27 0 27 54
16552024-02-20T22:42:27.109Z Skipped 0 0 0 0
16562024-02-20T22:42:27.109Z Error 0 0 0 0
16572024-02-20T22:42:27.109ZLast Flush: 1024 1001 1024
16582024-02-20T22:42:27.110ZDownstairs last five completed: 1001 1000
16592024-02-20T22:42:27.110ZUpstairs last five completed: 30 29 28 27 26
16602024-02-20T22:42:27.110Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:26.607032309Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1191}
16612024-02-20T22:42:27.110ZWrote out file "/tmp/test_repair_verify.data"
16622024-02-20T22:42:27.121Z
16632024-02-20T22:42:27.121ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16642024-02-20T22:42:27.121ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16652024-02-20T22:42:27.121Z 0 000-019 12 12 12 22 22 22 F F F
16662024-02-20T22:42:27.121Z 1 020-039 12 12 12 23 23 23 F F F
16672024-02-20T22:42:27.122Z 2 040-059 10 10 10 20 20 20 F F F
16682024-02-20T22:42:27.122Z 3 060-079 5 5 5 8 8 8 F F F
16692024-02-20T22:42:27.122Z 4 080-099 10 10 10 20 20 20 F F F
16702024-02-20T22:42:27.122Z 5 100-119 14 12 14 29 22 29 F F F <---
16712024-02-20T22:42:27.122Z 6 120-139 8 8 8 14 14 14 T F T <---
16722024-02-20T22:42:27.123Z 7 140-159 12 12 12 23 23 23 T F T <---
16732024-02-20T22:42:27.123Z 8 160-179 13 13 13 24 24 24 F F F
16742024-02-20T22:42:27.123Z 9 180-199 14 13 14 27 24 27 T F T <---
16752024-02-20T22:42:27.123Z 10 200-219 14 1 14 29 1 29 T F T <---
16762024-02-20T22:42:27.123Z 11 220-239 14 13 14 29 24 29 F F F <---
16772024-02-20T22:42:27.124Z 12 240-259 2 2 2 2 2 2 F F F
16782024-02-20T22:42:27.124Z 13 260-279 4 4 4 7 7 7 F F F
16792024-02-20T22:42:27.124Z 14 280-299 4 4 4 5 5 5 F F F
16802024-02-20T22:42:27.124Z 15 300-319 13 13 13 24 24 24 F F F
16812024-02-20T22:42:27.124Z 16 320-339 6 6 6 9 9 9 F F F
16822024-02-20T22:42:27.124Z 17 340-359 6 6 6 9 9 9 F F F
16832024-02-20T22:42:27.125Z 18 360-379 14 13 14 27 24 27 T F T <---
16842024-02-20T22:42:27.125Z 19 380-399 12 12 12 23 23 23 F F F
16852024-02-20T22:42:27.125Z 20 400-419 12 12 12 23 23 23 F F F
16862024-02-20T22:42:27.125Z 21 420-439 14 12 14 27 22 27 F F F <---
16872024-02-20T22:42:27.125Z 22 440-459 14 13 14 27 24 27 F F F <---
16882024-02-20T22:42:27.125Z 23 460-479 13 13 13 24 24 24 F F F
16892024-02-20T22:42:27.126Z 24 480-499 12 12 12 23 23 23 F F F
16902024-02-20T22:42:27.126Z 25 500-519 11 11 11 21 21 21 F F F
16912024-02-20T22:42:27.126Z 26 520-539 11 11 11 21 21 21 F F F
16922024-02-20T22:42:27.126Z 27 540-559 14 10 14 27 20 27 F F F <---
16932024-02-20T22:42:27.126Z 28 560-579 12 12 12 23 23 23 F F F
16942024-02-20T22:42:27.127Z 29 580-599 14 12 14 27 23 27 F F F <---
16952024-02-20T22:42:27.127ZMax gen: 14, Max flush: 29
16962024-02-20T22:42:27.127ZError: Difference in extent metadata found!
16972024-02-20T22:42:27.127ZA Difference in extent metadata is expected here
16982024-02-20T22:42:27.127ZOn loop 7
16992024-02-20T22:42:27.128Z
17002024-02-20T22:42:27.128ZVerifying data now
17012024-02-20T22:42:32.219ZNo change after verify
17022024-02-20T22:42:32.220ZLoop: 7 Downstairs dump after verify (and repair):
17032024-02-20T22:42:32.236ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17042024-02-20T22:42:32.236Z 0 000-019 12 12 12 22 22 22 F F F
17052024-02-20T22:42:32.236Z 1 020-039 12 12 12 23 23 23 F F F
17062024-02-20T22:42:32.236Z 2 040-059 10 10 10 20 20 20 F F F
17072024-02-20T22:42:32.236Z 3 060-079 5 5 5 8 8 8 F F F
17082024-02-20T22:42:32.236Z 4 080-099 10 10 10 20 20 20 F F F
17092024-02-20T22:42:32.236Z 5 100-119 14 14 14 29 29 29 F F F
17102024-02-20T22:42:32.237Z 6 120-139 15 15 15 30 30 30 F F F
17112024-02-20T22:42:32.237Z 7 140-159 15 15 15 30 30 30 F F F
17122024-02-20T22:42:32.237Z 8 160-179 13 13 13 24 24 24 F F F
17132024-02-20T22:42:32.237Z 9 180-199 15 15 15 30 30 30 F F F
17142024-02-20T22:42:32.237Z 10 200-219 15 15 15 30 30 30 F F F
17152024-02-20T22:42:32.237Z 11 220-239 14 14 14 29 29 29 F F F
17162024-02-20T22:42:32.237Z 12 240-259 2 2 2 2 2 2 F F F
17172024-02-20T22:42:32.237Z 13 260-279 4 4 4 7 7 7 F F F
17182024-02-20T22:42:32.237Z 14 280-299 4 4 4 5 5 5 F F F
17192024-02-20T22:42:32.237Z 15 300-319 13 13 13 24 24 24 F F F
17202024-02-20T22:42:32.237Z 16 320-339 6 6 6 9 9 9 F F F
17212024-02-20T22:42:32.237Z 17 340-359 6 6 6 9 9 9 F F F
17222024-02-20T22:42:32.237Z 18 360-379 15 15 15 30 30 30 F F F
17232024-02-20T22:42:32.237Z 19 380-399 12 12 12 23 23 23 F F F
17242024-02-20T22:42:32.237Z 20 400-419 12 12 12 23 23 23 F F F
17252024-02-20T22:42:32.237Z 21 420-439 14 14 14 27 27 27 F F F
17262024-02-20T22:42:32.237Z 22 440-459 14 14 14 27 27 27 F F F
17272024-02-20T22:42:32.237Z 23 460-479 13 13 13 24 24 24 F F F
17282024-02-20T22:42:32.237Z 24 480-499 12 12 12 23 23 23 F F F
17292024-02-20T22:42:32.237Z 25 500-519 11 11 11 21 21 21 F F F
17302024-02-20T22:42:32.237Z 26 520-539 11 11 11 21 21 21 F F F
17312024-02-20T22:42:32.237Z 27 540-559 14 14 14 27 27 27 F F F
17322024-02-20T22:42:32.237Z 28 560-579 12 12 12 23 23 23 F F F
17332024-02-20T22:42:32.237Z 29 580-599 14 14 14 27 27 27 F F F
17342024-02-20T22:42:32.237ZMax gen: 15, Max flush: 30
17352024-02-20T22:42:32.238Z
17362024-02-20T22:42:32.238ZBegin loop 8
17372024-02-20T22:42:32.238ZDownstairs to restart: 1
17382024-02-20T22:42:32.246Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.785768308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199}
17392024-02-20T22:42:32.381Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786099266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199}
17402024-02-20T22:42:32.381ZCrucible runtime is spawned
17412024-02-20T22:42:32.381Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786145975Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199}
17422024-02-20T22:42:32.381Z{"msg":"Crucible fe0a09e2-b241-4f54-9369-a035ba74fa2a has session id: 660bf73c-5623-48db-87dd-a7eafefaea9f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786178453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17432024-02-20T22:42:32.381Z{"msg":"Upstairs opts: Upstairs UUID: fe0a09e2-b241-4f54-9369-a035ba74fa2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786229152Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17442024-02-20T22:42:32.381Z{"msg":"Crucible stats registered with UUID: fe0a09e2-b241-4f54-9369-a035ba74fa2a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.78628127Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17452024-02-20T22:42:32.381Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786306519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199}
17462024-02-20T22:42:32.381Z{"msg":"fe0a09e2-b241-4f54-9369-a035ba74fa2a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786353327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17472024-02-20T22:42:32.381Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786383296Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17482024-02-20T22:42:32.381Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786428084Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17492024-02-20T22:42:32.381Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786473853Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17502024-02-20T22:42:32.381Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786521501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17512024-02-20T22:42:32.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.78656611Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17522024-02-20T22:42:32.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786608988Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17532024-02-20T22:42:32.382Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786652856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17542024-02-20T22:42:32.382Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786689145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17552024-02-20T22:42:32.382Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786707885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17562024-02-20T22:42:32.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786917367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17572024-02-20T22:42:32.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.786970405Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17582024-02-20T22:42:32.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.787010874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17592024-02-20T22:42:32.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.787624232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17602024-02-20T22:42:32.382Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.78768757Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17612024-02-20T22:42:32.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.787825805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17622024-02-20T22:42:32.382Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.787863823Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17632024-02-20T22:42:32.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.787935691Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17642024-02-20T22:42:32.382Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788026147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17652024-02-20T22:42:32.382Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788194361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17662024-02-20T22:42:32.382Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788500041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17672024-02-20T22:42:32.382Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788537079Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17682024-02-20T22:42:32.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788766861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17692024-02-20T22:42:32.382Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788807019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17702024-02-20T22:42:32.382Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788833958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17712024-02-20T22:42:32.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.788987463Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17722024-02-20T22:42:32.382Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789040271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17732024-02-20T22:42:32.382Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.78906626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17742024-02-20T22:42:32.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789093669Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17752024-02-20T22:42:32.382Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789118028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17762024-02-20T22:42:32.383Z{"msg":"[0]R flush_numbers[0..12]: [22, 23, 20, 8, 20, 29, 30, 30, 24, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789146077Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17772024-02-20T22:42:32.383Z{"msg":"[0]R generation[0..12]: [12, 12, 10, 5, 10, 14, 15, 15, 13, 15, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789179606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17782024-02-20T22:42:32.383Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789213525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17792024-02-20T22:42:32.383Z{"msg":"[1]R flush_numbers[0..12]: [22, 23, 20, 8, 20, 29, 30, 30, 24, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789239964Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
17802024-02-20T22:42:32.383Z,"pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17812024-02-20T22:42:32.383Z{"msg":"[1]R generation[0..12]: [12, 12, 10, 5, 10, 14, 15, 15, 13, 15, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789278353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17822024-02-20T22:42:32.383Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789313161Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17832024-02-20T22:42:32.383Z{"msg":"[2]R flush_numbers[0..12]: [22, 23, 20, 8, 20, 29, 30, 30, 24, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.78933971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17842024-02-20T22:42:32.383Z{"msg":"[2]R generation[0..12]: [12, 12, 10, 5, 10, 14, 15, 15, 13, 15, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789372269Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17852024-02-20T22:42:32.383Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789398868Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17862024-02-20T22:42:32.383Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789429387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17872024-02-20T22:42:32.383Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789455496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17882024-02-20T22:42:32.383Z{"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789478326Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17892024-02-20T22:42:32.383Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789509434Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17902024-02-20T22:42:32.383Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789533824Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17912024-02-20T22:42:32.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789556003Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17922024-02-20T22:42:32.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789583412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17932024-02-20T22:42:32.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789607531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17942024-02-20T22:42:32.383Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.78963444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17952024-02-20T22:42:32.383Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789656859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17962024-02-20T22:42:32.383ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17972024-02-20T22:42:32.383Z{"msg":"fe0a09e2-b241-4f54-9369-a035ba74fa2a is now active with session: 660bf73c-5623-48db-87dd-a7eafefaea9f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789687198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17982024-02-20T22:42:32.383Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789716937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199,"session_id":"660bf73c-5623-48db-87dd-a7eafefaea9f"}
17992024-02-20T22:42:32.383Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:31.789739346Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199}
18002024-02-20T22:42:32.384ZLoading write count information from file "/tmp/test_repair_verify.data"
18012024-02-20T22:42:32.384ZRun Repair workload
18022024-02-20T22:42:32.384Z01/30 Write block 108 len 28672 data: 3 4 4 5 6 7 5
18032024-02-20T22:42:32.384Z02/30 Flush
18042024-02-20T22:42:32.415Z03/30 Write block 127 len 4096 data: 3
18052024-02-20T22:42:32.416Z04/30 Read block 522 len 4096
18062024-02-20T22:42:32.416Z05/30 Write block 371 len 8192 data: 3 4
18072024-02-20T22:42:32.416Z06/30 Read block 484 len 24576
18082024-02-20T22:42:32.416Z07/30 Read block 24 len 20480
18092024-02-20T22:42:32.416Z08/30 Flush
18102024-02-20T22:42:32.416Z09/30 Write block 412 len 8192 data: 3 3
18112024-02-20T22:42:32.416Z10/30 Write block 253 len 8192 data: 2 2
18122024-02-20T22:42:32.416Z11/30 Flush
18132024-02-20T22:42:32.416Z12/30 Write block 58 len 8192 data: 2 2
18142024-02-20T22:42:32.416Z13/30 Read block 546 len 36864
18152024-02-20T22:42:32.416Z14/30 Write block 25 len 36864 data: 3 3 3 4 3 4 4 4 4
18162024-02-20T22:42:32.416Z15/30 Write block 325 len 24576 data: 2 2 2 2 2 2
18172024-02-20T22:42:32.416Z16/30 Flush
18182024-02-20T22:42:32.416Z17/30 Write block 588 len 40960 data: 4 4 4 3 3 2 2 2 2 2
18192024-02-20T22:42:32.416Z18/30 Read block 127 len 24576
18202024-02-20T22:42:32.416Z19/30 Read block 337 len 8192
18212024-02-20T22:42:32.416Z20/30 Read block 391 len 16384
18222024-02-20T22:42:32.416Z21/30 Read block 43 len 16384
18232024-02-20T22:42:32.416Z22/30 Read block 250 len 40960
18242024-02-20T22:42:32.416Z23/30 Read block 192 len 36864
18252024-02-20T22:42:32.416Z24/30 Read block 514 len 20480
18262024-02-20T22:42:32.416Z25/30 Flush
18272024-02-20T22:42:32.416Z26/30 Write block 377 len 4096 data: 3
18282024-02-20T22:42:32.416Z27/30 Read block 489 len 40960
18292024-02-20T22:42:32.417Z28/30 Read block 597 len 8192
18302024-02-20T22:42:32.417Z29/30 Read block 518 len 28672
18312024-02-20T22:42:32.417Z30/30 Read block 454 len 28672
18322024-02-20T22:42:32.417Z----------------------------------------------------------------
18332024-02-20T22:42:32.417Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18342024-02-20T22:42:32.417ZStates: Active Active Active
18352024-02-20T22:42:32.417ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18362024-02-20T22:42:32.417Z 1 Acked 1000 Write 7 Done Sent Done false
18372024-02-20T22:42:32.417Z 2 Acked 1001 Flush 0 Done Sent Done false
18382024-02-20T22:42:32.417Z 3 Acked 1002 Write 1 Done Sent Done false
18392024-02-20T22:42:32.417Z 4 Acked 1003 Read 1 Done Sent Done false
18402024-02-20T22:42:32.417Z 5 Acked 1004 Write 2 Done Sent Done false
18412024-02-20T22:42:32.417Z 6 Acked 1005 Read 6 Done Sent Done false
18422024-02-20T22:42:32.417Z 7 Acked 1006 Read 5 Done Sent Done false
18432024-02-20T22:42:32.417Z 8 Acked 1007 Flush 0 Done Sent Done false
18442024-02-20T22:42:32.417Z 9 Acked 1008 Write 2 Done Sent Done false
18452024-02-20T22:42:32.417Z 10 Acked 1009 Write 2 Done Sent Done false
18462024-02-20T22:42:32.417Z 11 Acked 1010 Flush 0 Done Sent Done false
18472024-02-20T22:42:32.417Z 12 Acked 1011 Write 2 Done Sent Done false
18482024-02-20T22:42:32.417Z 13 Acked 1012 Read 9 Done Sent Done false
18492024-02-20T22:42:32.417Z 14 Acked 1013 Write 9 Done Sent Done false
18502024-02-20T22:42:32.417Z 15 Acked 1014 Write 6 Done Sent Done false
18512024-02-20T22:42:32.418Z 16 Acked 1015 Flush 0 Done Sent Done false
18522024-02-20T22:42:32.418Z 17 Acked 1016 Write 10 Done Sent Done false
18532024-02-20T22:42:32.418Z 18 Acked 1017 Read 6 Done Sent Done false
18542024-02-20T22:42:32.418Z 19 Acked 1018 Read 2 Done Sent Done false
18552024-02-20T22:42:32.418Z 20 Acked 1019 Read 4 Done Sent Done false
18562024-02-20T22:42:32.418Z 21 Acked 1020 Read 4 Done Sent Done false
18572024-02-20T22:42:32.418Z 22 Acked 1021 Read 10 Done Sent Done false
18582024-02-20T22:42:32.418Z 23 Acked 1022 Read 9 Done Sent Done false
18592024-02-20T22:42:32.418Z 24 Acked 1023 Read 5 Done Sent Done false
18602024-02-20T22:42:32.418Z 25 Acked 1024 Flush 0 Done Sent Done false
18612024-02-20T22:42:32.418Z 26 Acked 1025 Write 1 Done Sent Done false
18622024-02-20T22:42:32.418Z 27 Acked 1026 Read 10 Done Sent Done false
18632024-02-20T22:42:32.418Z 28 Acked 1027 Read 2 Done Sent Done false
18642024-02-20T22:42:32.418Z 29 Acked 1028 Read 7 Done Sent Done false
18652024-02-20T22:42:32.418Z 30 Acked 1029 Read 7 Sent Sent Done false
18662024-02-20T22:42:32.418Z STATES DS:0 DS:1 DS:2 TOTAL
18672024-02-20T22:42:32.418Z New 0 0 0 0
18682024-02-20T22:42:32.418Z Sent 1 30 0 31
18692024-02-20T22:42:32.418Z Done 29 0 30 59
18702024-02-20T22:42:32.418Z Skipped 0 0 0 0
18712024-02-20T22:42:32.418Z Error 0 0 0 0
18722024-02-20T22:42:32.418ZLast Flush: 1024 0 1024
18732024-02-20T22:42:32.418ZDownstairs last five completed:
18742024-02-20T22:42:32.418ZUpstairs last five completed: 30 29 28 27 26
18752024-02-20T22:42:32.418Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:31.86452146Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1199}
18762024-02-20T22:42:32.418ZWrote out file "/tmp/test_repair_verify.data"
18772024-02-20T22:42:32.419Z
18782024-02-20T22:42:32.419ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18792024-02-20T22:42:32.419ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18802024-02-20T22:42:32.419Z 0 000-019 12 12 12 22 22 22 F F F
18812024-02-20T22:42:32.419Z 1 020-039 16 12 16 34 23 34 F F F <---
18822024-02-20T22:42:32.419Z 2 040-059 16 10 16 34 20 34 F F F <---
18832024-02-20T22:42:32.419Z 3 060-079 5 5 5 8 8 8 F F F
18842024-02-20T22:42:32.419Z 4 080-099 10 10 10 20 20 20 F F F
18852024-02-20T22:42:32.419Z 5 100-119 16 14 16 31 29 31 F F F <---
18862024-02-20T22:42:32.419Z 6 120-139 16 15 16 32 30 32 F F F <---
18872024-02-20T22:42:32.419Z 7 140-159 15 15 15 30 30 30 F F F
18882024-02-20T22:42:32.419Z 8 160-179 13 13 13 24 24 24 F F F
18892024-02-20T22:42:32.419Z 9 180-199 15 15 15 30 30 30 F F F
18902024-02-20T22:42:32.419Z 10 200-219 15 15 15 30 30 30 F F F
18912024-02-20T22:42:32.419Z 11 220-239 14 14 14 29 29 29 F F F
18922024-02-20T22:42:32.419Z 12 240-259 16 2 16 33 2 33 F F F <---
18932024-02-20T22:42:32.419Z 13 260-279 4 4 4 7 7 7 F F F
18942024-02-20T22:42:32.419Z 14 280-299 4 4 4 5 5 5 F F F
18952024-02-20T22:42:32.419Z 15 300-319 13 13 13 24 24 24 F F F
18962024-02-20T22:42:32.419Z 16 320-339 16 6 16 34 9 34 F F F <---
18972024-02-20T22:42:32.419Z 17 340-359 6 6 6 9 9 9 F F F
18982024-02-20T22:42:32.419Z 18 360-379 16 15 16 32 30 32 T F T <---
18992024-02-20T22:42:32.419Z 19 380-399 12 12 12 23 23 23 F F F
19002024-02-20T22:42:32.419Z 20 400-419 16 12 16 33 23 33 F F F <---
19012024-02-20T22:42:32.419Z 21 420-439 14 14 14 27 27 27 F F F
19022024-02-20T22:42:32.420Z 22 440-459 14 14 14 27 27 27 F F F
19032024-02-20T22:42:32.420Z 23 460-479 13 13 13 24 24 24 F F F
19042024-02-20T22:42:32.443Z 24 480-499 12 12 12 23 23 23 F F F
19052024-02-20T22:42:32.443Z 25 500-519 11 11 11 21 21 21 F F F
19062024-02-20T22:42:32.443Z 26 520-539 11 11 11 21 21 21 F F F
19072024-02-20T22:42:32.443Z 27 540-559 14 14 14 27 27 27 F F F
19082024-02-20T22:42:32.443Z 28 560-579 12 12 12 23 23 23 F F F
19092024-02-20T22:42:32.443Z 29 580-599 16 14 16 35 27 35 F F F <---
19102024-02-20T22:42:32.443ZMax gen: 16, Max flush: 35
19112024-02-20T22:42:32.443ZError: Difference in extent metadata found!
19122024-02-20T22:42:32.443ZA Difference in extent metadata is expected here
19132024-02-20T22:42:32.443ZOn loop 8
19142024-02-20T22:42:32.443Z
19152024-02-20T22:42:32.443ZVerifying data now
19162024-02-20T22:42:37.304ZNo change after verify
19172024-02-20T22:42:37.304ZLoop: 8 Downstairs dump after verify (and repair):
19182024-02-20T22:42:37.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19192024-02-20T22:42:37.313Z 0 000-019 12 12 12 22 22 22 F F F
19202024-02-20T22:42:37.314Z 1 020-039 16 16 16 34 34 34 F F F
19212024-02-20T22:42:37.314Z 2 040-059 16 16 16 34 34 34 F F F
19222024-02-20T22:42:37.314Z 3 060-079 5 5 5 8 8 8 F F F
19232024-02-20T22:42:37.314Z 4 080-099 10 10 10 20 20 20 F F F
19242024-02-20T22:42:37.314Z 5 100-119 16 16 16 31 31 31 F F F
19252024-02-20T22:42:37.314Z 6 120-139 16 16 16 32 32 32 F F F
19262024-02-20T22:42:37.314Z 7 140-159 15 15 15 30 30 30 F F F
19272024-02-20T22:42:37.314Z 8 160-179 13 13 13 24 24 24 F F F
19282024-02-20T22:42:37.314Z 9 180-199 15 15 15 30 30 30 F F F
19292024-02-20T22:42:37.314Z 10 200-219 15 15 15 30 30 30 F F F
19302024-02-20T22:42:37.314Z 11 220-239 14 14 14 29 29 29 F F F
19312024-02-20T22:42:37.314Z 12 240-259 16 16 16 33 33 33 F F F
19322024-02-20T22:42:37.314Z 13 260-279 4 4 4 7 7 7 F F F
19332024-02-20T22:42:37.314Z 14 280-299 4 4 4 5 5 5 F F F
19342024-02-20T22:42:37.318Z 15 300-319 13 13 13 24 24 24 F F F
19352024-02-20T22:42:37.319Z 16 320-339 16 16 16 34 34 34 F F F
19362024-02-20T22:42:37.319Z 17 340-359 6 6 6 9 9 9 F F F
19372024-02-20T22:42:37.319Z 18 360-379 17 17 17 36 36 36 F F F
19382024-02-20T22:42:37.319Z 19 380-399 12 12 12 23 23 23 F F F
19392024-02-20T22:42:37.319Z 20 400-419 16 16 16 33 33 33 F F F
19402024-02-20T22:42:37.319Z 21 420-439 14 14 14 27 27 27 F F F
19412024-02-20T22:42:37.319Z 22 440-459 14 14 14 27 27 27 F F F
19422024-02-20T22:42:37.319Z 23 460-479 13 13 13 24 24 24 F F F
19432024-02-20T22:42:37.319Z 24 480-499 12 12 12 23 23 23 F F F
19442024-02-20T22:42:37.319Z 25 500-519 11 11 11 21 21 21 F F F
19452024-02-20T22:42:37.319Z 26 520-539 11 11 11 21 21 21 F F F
19462024-02-20T22:42:37.319Z 27 540-559 14 14 14 27 27 27 F F F
19472024-02-20T22:42:37.319Z 28 560-579 12 12 12 23 23 23 F F F
19482024-02-20T22:42:37.319Z 29 580-599 16 16 16 35 35 35 F F F
19492024-02-20T22:42:37.319ZMax gen: 17, Max flush: 36
19502024-02-20T22:42:37.319Z
19512024-02-20T22:42:37.319ZBegin loop 9
19522024-02-20T22:42:37.319ZDownstairs to restart: 2
19532024-02-20T22:42:37.335Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.874787568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207}
19542024-02-20T22:42:37.335Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875112396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207}
19552024-02-20T22:42:37.343Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875187653Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207}
19562024-02-20T22:42:37.343Z{"msg":"Crucible 4df8b3d8-2126-4818-93b1-0e22925a1f39 has session id: 72c115c8-c114-495a-a43b-42d433fc0893","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875252841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":Crucible runtime is spawned
19572024-02-20T22:42:37.343Z"72c115c8-c114-495a-a43b-42d433fc0893"}
19582024-02-20T22:42:37.343Z{"msg":"Upstairs opts: Upstairs UUID: 4df8b3d8-2126-4818-93b1-0e22925a1f39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875342828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19592024-02-20T22:42:37.343Z{"msg":"Crucible stats registered with UUID: 4df8b3d8-2126-4818-93b1-0e22925a1f39","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875374457Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19602024-02-20T22:42:37.344Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875389896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207}
19612024-02-20T22:42:37.344Z{"msg":"4df8b3d8-2126-4818-93b1-0e22925a1f39 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875490413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19622024-02-20T22:42:37.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875516042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19632024-02-20T22:42:37.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875531271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19642024-02-20T22:42:37.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875545201Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19652024-02-20T22:42:37.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.87556716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19662024-02-20T22:42:37.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875591009Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19672024-02-20T22:42:37.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875616388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19682024-02-20T22:42:37.344Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875645437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19692024-02-20T22:42:37.344Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875681856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19702024-02-20T22:42:37.344Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875725774Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19712024-02-20T22:42:37.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.875891768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19722024-02-20T22:42:37.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876152769Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19732024-02-20T22:42:37.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876207667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19742024-02-20T22:42:37.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876582354Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19752024-02-20T22:42:37.344Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876620262Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19762024-02-20T22:42:37.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876759167Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19772024-02-20T22:42:37.344Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876809625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19782024-02-20T22:42:37.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876852274Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19792024-02-20T22:42:37.344Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.876888183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19802024-02-20T22:42:37.344Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877206811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19812024-02-20T22:42:37.344Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877314697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19822024-02-20T22:42:37.344Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877550659Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19832024-02-20T22:42:37.344Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877709913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19842024-02-20T22:42:37.344Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877770591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19852024-02-20T22:42:37.345Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.87780904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19862024-02-20T22:42:37.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877839549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19872024-02-20T22:42:37.345Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877862168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19882024-02-20T22:42:37.345Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877876357Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19892024-02-20T22:42:37.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.877983423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19902024-02-20T22:42:37.345Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878025162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19912024-02-20T22:42:37.345Z{"msg":"[0]R flush_numbers[0..12]: [22, 34, 34, 8, 20, 31, 32, 30, 24, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878060481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19922024-02-20T22:42:37.345Z{"msg":"[0]R generation[0..12]: [12, 16, 16, 5, 10, 16, 16, 15, 13, 15, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.87807714Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19932024-02-20T22:42:37.345Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878098949Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19942024-02-20T22:42:37.345Z{"msg":"[1]R flush_numbers[0..12]: [22, 34, 34, 8, 20, 31, 32, 30, 24, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878113369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19952024-02-20T22:42:37.345Z{"msg":"[1]R generation[0..12]: [12, 16, 16, 5, 10, 16, 16, 15, 13, 15, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878140488Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19962024-02-20T22:42:37.345Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878155497Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19972024-02-20T22:42:37.345ZWait for a query_work_queue command to finish before sending IO
19982024-02-20T22:42:37.345Z{"msg":"[2]R flush_numbers[0..12]: [22, 34, 34, 8, 20, 31, 32, 30, 24, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878181506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
19992024-02-20T22:42:37.345Z{"msg":"[2]R generation[0..12]: [12, 16, 16, 5, 10, 16, 16, 15, 13, 15, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878195506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20002024-02-20T22:42:37.345Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878216635Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20012024-02-20T22:42:37.345Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878230224Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20022024-02-20T22:42:37.345Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878265883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20032024-02-20T22:42:37.345Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878290062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20042024-02-20T22:42:37.345Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878335721Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20052024-02-20T22:42:37.345Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878374859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20062024-02-20T22:42:37.345Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878406338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20072024-02-20T22:42:37.345Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878442757Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20082024-02-20T22:42:37.345Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878478476Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20092024-02-20T22:42:37.345Zip-10-150-1-191.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20102024-02-20T22:42:37.346Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878517544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20112024-02-20T22:42:37.346Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878533244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20122024-02-20T22:42:37.346Z{"msg":"4df8b3d8-2126-4818-93b1-0e22925a1f39 is now active with session: 72c115c8-c114-495a-a43b-42d433fc0893","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878555493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20132024-02-20T22:42:37.346Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878568243Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207,"session_id":"72c115c8-c114-495a-a43b-42d433fc0893"}
20142024-02-20T22:42:37.346Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:36.878587422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207}
20152024-02-20T22:42:37.346ZLoading write count information from file "/tmp/test_repair_verify.data"
20162024-02-20T22:42:37.346ZRun Repair workload
20172024-02-20T22:42:37.346Z01/30 Write block 68 len 28672 data: 2 3 3 2 2 2 2
20182024-02-20T22:42:37.350Z02/30 Read block 579 len 24576
20192024-02-20T22:42:37.350Z03/30 Write block 116 len 40960 data: 5 5 2 2 2 2 2 2 2 2
20202024-02-20T22:42:37.350Z04/30 Read block 33 len 16384
20212024-02-20T22:42:37.350Z05/30 Read block 310 len 16384
20222024-02-20T22:42:37.350Z06/30 Flush
20232024-02-20T22:42:37.380Z07/30 Write block 550 len 12288 data: 3 2 2
20242024-02-20T22:42:37.386Z08/30 Read block 378 len 40960
20252024-02-20T22:42:37.386Z09/30 Write block 409 len 32768 data: 4 4 4 4 4 4 4 4
20262024-02-20T22:42:37.386Z10/30 Read block 441 len 24576
20272024-02-20T22:42:37.389Z11/30 Read block 51 len 36864
20282024-02-20T22:42:37.389Z12/30 Read block 196 len 36864
20292024-02-20T22:42:37.389Z13/30 Read block 478 len 16384
20302024-02-20T22:42:37.389Z14/30 Read block 284 len 16384
20312024-02-20T22:42:37.389Z15/30 Read block 527 len 8192
20322024-02-20T22:42:37.390Z16/30 Read block 422 len 4096
20332024-02-20T22:42:37.394Z17/30 Write block 36 len 28672 data: 3 3 3 3 2 2 2
20342024-02-20T22:42:37.394Z18/30 Write block 594 len 24576 data: 3 3 3 3 2 2
20352024-02-20T22:42:37.394Z19/30 Write block 521 len 8192 data: 2 4
20362024-02-20T22:42:37.394Z20/30 Read block 165 len 32768
20372024-02-20T22:42:37.394Z21/30 Read block 104 len 36864
20382024-02-20T22:42:37.398Z22/30 Write block 150 len 12288 data: 4 4 3
20392024-02-20T22:42:37.398Z23/30 Write block 135 len 16384 data: 3 3 3 4
20402024-02-20T22:42:37.398Z24/30 Read block 319 len 28672
20412024-02-20T22:42:37.398Z25/30 Read block 349 len 36864
20422024-02-20T22:42:37.398Z26/30 Write block 418 len 12288 data: 3 3 3
20432024-02-20T22:42:37.398Z27/30 Read block 420 len 20480
20442024-02-20T22:42:37.403Z28/30 Write block 99 len 16384 data: 4 4 3 2
20452024-02-20T22:42:37.403Z29/30 Read block 450 len 24576
20462024-02-20T22:42:37.403Z30/30 Write block 174 len 32768 data: 2 3 3 3 3 4 5 5
20472024-02-20T22:42:37.403Z----------------------------------------------------------------
20482024-02-20T22:42:37.403Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20492024-02-20T22:42:37.403ZStates: Active Active Active
20502024-02-20T22:42:37.403ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20512024-02-20T22:42:37.403Z 1 Acked 1000 Write 7 Done Done Done false
20522024-02-20T22:42:37.403Z 2 Acked 1001 Read 6 Done Done Done false
20532024-02-20T22:42:37.403Z 3 Acked 1002 Write 10 Done Done Sent false
20542024-02-20T22:42:37.403Z 4 Acked 1003 Read 4 Done Done Sent false
20552024-02-20T22:42:37.403Z 5 Acked 1004 Read 4 Done Done Sent false
20562024-02-20T22:42:37.404Z 6 Acked 1005 Flush 0 Done Done Sent false
20572024-02-20T22:42:37.404Z 7 Acked 1006 Write 3 Done Done Sent false
20582024-02-20T22:42:37.404Z 8 Acked 1007 Read 10 Done Done Sent false
20592024-02-20T22:42:37.404Z 9 Acked 1008 Write 8 Done Done Sent false
20602024-02-20T22:42:37.404Z 10 Acked 1009 Read 6 Done Done Sent false
20612024-02-20T22:42:37.404Z 11 Acked 1010 Read 9 Done Done Sent false
20622024-02-20T22:42:37.404Z 12 Acked 1011 Read 9 Done Done Sent false
20632024-02-20T22:42:37.404Z 13 Acked 1012 Read 4 Done Done Sent false
20642024-02-20T22:42:37.404Z 14 Acked 1013 Read 4 Done Done Sent false
20652024-02-20T22:42:37.404Z 15 Acked 1014 Read 2 Done Done Sent false
20662024-02-20T22:42:37.404Z 16 Acked 1015 Read 1 Done Done Sent false
20672024-02-20T22:42:37.404Z 17 Acked 1016 Write 7 Done Done Sent false
20682024-02-20T22:42:37.404Z 18 Acked 1017 Write 6 Done Done Sent false
20692024-02-20T22:42:37.404Z 19 Acked 1018 Write 2 Done Done Sent false
20702024-02-20T22:42:37.404Z 20 Acked 1019 Read 8 Done Done Sent false
20712024-02-20T22:42:37.404Z 21 Acked 1020 Read 9 Done Done Sent false
20722024-02-20T22:42:37.404Z 22 Acked 1021 Write 3 Done Done Sent false
20732024-02-20T22:42:37.404Z 23 Acked 1022 Write 4 Done Done Sent false
20742024-02-20T22:42:37.404Z 24 Acked 1023 Read 7 Done Done Sent false
20752024-02-20T22:42:37.404Z 25 Acked 1024 Read 9 Done Done Sent false
20762024-02-20T22:42:37.404Z 26 Acked 1025 Write 3 Done Done Sent false
20772024-02-20T22:42:37.404Z 27 Acked 1026 Read 5 Done Done Sent false
20782024-02-20T22:42:37.404Z 28 Acked 1027 Write 4 Done Done Sent false
20792024-02-20T22:42:37.404Z 29 Acked 1028 Read 6 Done Done Sent false
20802024-02-20T22:42:37.404Z 30 Acked 1029 Write 8 Sent Sent Sent false
20812024-02-20T22:42:37.404Z STATES DS:0 DS:1 DS:2 TOTAL
20822024-02-20T22:42:37.405Z New 0 0 0 0
20832024-02-20T22:42:37.405Z Sent 1 1 28 30
20842024-02-20T22:42:37.405Z Done 29 29 2 60
20852024-02-20T22:42:37.405Z Skipped 0 0 0 0
20862024-02-20T22:42:37.405Z Error 0 0 0 0
20872024-02-20T22:42:37.405ZLast Flush: 1005 1005 0
20882024-02-20T22:42:37.405ZDownstairs last five completed:
20892024-02-20T22:42:37.405ZUpstairs last five completed: 30 29 28 27 26
20902024-02-20T22:42:37.405Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:36.94206856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1207}
20912024-02-20T22:42:37.405ZWrote out file "/tmp/test_repair_verify.data"
20922024-02-20T22:42:37.409Z
20932024-02-20T22:42:37.409ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20942024-02-20T22:42:37.437ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20952024-02-20T22:42:37.438Z 0 000-019 12 12 12 22 22 22 F F F
20962024-02-20T22:42:37.438Z 1 020-039 16 16 16 34 34 34 T T F <---
20972024-02-20T22:42:37.438Z 2 040-059 16 16 16 34 34 34 T T F <---
20982024-02-20T22:42:37.438Z 3 060-079 18 18 5 37 37 8 F F T <---
20992024-02-20T22:42:37.438Z 4 080-099 10 10 10 20 20 20 T T F <---
21002024-02-20T22:42:37.438Z 5 100-119 18 18 16 37 37 31 T T F <---
21012024-02-20T22:42:37.438Z 6 120-139 18 18 16 37 37 32 T T F <---
21022024-02-20T22:42:37.438Z 7 140-159 15 15 15 30 30 30 T T F <---
21032024-02-20T22:42:37.438Z 8 160-179 13 13 13 24 24 24 T T F <---
21042024-02-20T22:42:37.463Z 9 180-199 15 15 15 30 30 30 T T F <---
21052024-02-20T22:42:37.464Z 10 200-219 15 15 15 30 30 30 F F F
21062024-02-20T22:42:37.464Z 11 220-239 14 14 14 29 29 29 F F F
21072024-02-20T22:42:37.464Z 12 240-259 16 16 16 33 33 33 F F F
21082024-02-20T22:42:37.465Z 13 260-279 4 4 4 7 7 7 F F F
21092024-02-20T22:42:37.465Z 14 280-299 4 4 4 5 5 5 F F F
21102024-02-20T22:42:37.465Z 15 300-319 13 13 13 24 24 24 F F F
21112024-02-20T22:42:37.465Z 16 320-339 16 16 16 34 34 34 F F F
21122024-02-20T22:42:37.465Z 17 340-359 6 6 6 9 9 9 F F F
21132024-02-20T22:42:37.466Z 18 360-379 17 17 17 36 36 36 F F F
21142024-02-20T22:42:37.466Z 19 380-399 12 12 12 23 23 23 F F F
21152024-02-20T22:42:37.466Z 20 400-419 16 16 16 33 33 33 T T F <---
21162024-02-20T22:42:37.466Z 21 420-439 14 14 14 27 27 27 T T F <---
21172024-02-20T22:42:37.466Z 22 440-459 14 14 14 27 27 27 F F F
21182024-02-20T22:42:37.467Z 23 460-479 13 13 13 24 24 24 F F F
21192024-02-20T22:42:37.467Z 24 480-499 12 12 12 23 23 23 F F F
21202024-02-20T22:42:37.467Z 25 500-519 11 11 11 21 21 21 F F F
21212024-02-20T22:42:37.467Z 26 520-539 11 11 11 21 21 21 T T F <---
21222024-02-20T22:42:37.468Z 27 540-559 14 14 14 27 27 27 T T F <---
21232024-02-20T22:42:37.468Z 28 560-579 12 12 12 23 23 23 F F F
21242024-02-20T22:42:37.468Z 29 580-599 16 16 16 35 35 35 T T F <---
21252024-02-20T22:42:37.468ZMax gen: 18, Max flush: 37
21262024-02-20T22:42:37.468ZError: Difference in extent metadata found!
21272024-02-20T22:42:37.468ZA Difference in extent metadata is expected here
21282024-02-20T22:42:37.469ZOn loop 9
21292024-02-20T22:42:37.469Z
21302024-02-20T22:42:37.469ZVerifying data now
21312024-02-20T22:42:38.969ZNo change after verify
21322024-02-20T22:42:38.969ZLoop: 9 Downstairs dump after verify (and repair):
21332024-02-20T22:42:38.974ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21342024-02-20T22:42:38.974Z 0 000-019 12 12 12 22 22 22 F F F
21352024-02-20T22:42:38.974Z 1 020-039 19 19 19 38 38 38 F F F
21362024-02-20T22:42:38.974Z 2 040-059 19 19 19 38 38 38 F F F
21372024-02-20T22:42:38.974Z 3 060-079 18 18 18 37 37 37 F F F
21382024-02-20T22:42:38.974Z 4 080-099 19 19 19 38 38 38 F F F
21392024-02-20T22:42:38.978Z 5 100-119 19 19 19 38 38 38 F F F
21402024-02-20T22:42:38.978Z 6 120-139 19 19 19 38 38 38 F F F
21412024-02-20T22:42:38.978Z 7 140-159 19 19 19 38 38 38 F F F
21422024-02-20T22:42:38.978Z 8 160-179 19 19 19 38 38 38 F F F
21432024-02-20T22:42:38.978Z 9 180-199 19 19 19 38 38 38 F F F
21442024-02-20T22:42:38.978Z 10 200-219 15 15 15 30 30 30 F F F
21452024-02-20T22:42:38.978Z 11 220-239 14 14 14 29 29 29 F F F
21462024-02-20T22:42:38.978Z 12 240-259 16 16 16 33 33 33 F F F
21472024-02-20T22:42:38.978Z 13 260-279 4 4 4 7 7 7 F F F
21482024-02-20T22:42:38.978Z 14 280-299 4 4 4 5 5 5 F F F
21492024-02-20T22:42:38.978Z 15 300-319 13 13 13 24 24 24 F F F
21502024-02-20T22:42:38.978Z 16 320-339 16 16 16 34 34 34 F F F
21512024-02-20T22:42:38.978Z 17 340-359 6 6 6 9 9 9 F F F
21522024-02-20T22:42:38.979Z 18 360-379 17 17 17 36 36 36 F F F
21532024-02-20T22:42:38.979Z 19 380-399 12 12 12 23 23 23 F F F
21542024-02-20T22:42:38.979Z 20 400-419 19 19 19 38 38 38 F F F
21552024-02-20T22:42:38.979Z 21 420-439 19 19 19 38 38 38 F F F
21562024-02-20T22:42:38.979Z 22 440-459 14 14 14 27 27 27 F F F
21572024-02-20T22:42:38.979Z 23 460-479 13 13 13 24 24 24 F F F
21582024-02-20T22:42:38.979Z 24 480-499 12 12 12 23 23 23 F F F
21592024-02-20T22:42:38.979Z 25 500-519 11 11 11 21 21 21 F F F
21602024-02-20T22:42:38.979Z 26 520-539 19 19 19 38 38 38 F F F
21612024-02-20T22:42:38.979Z 27 540-559 19 19 19 38 38 38 F F F
21622024-02-20T22:42:38.979Z 28 560-579 12 12 12 23 23 23 F F F
21632024-02-20T22:42:38.979Z 29 580-599 19 19 19 38 38 38 F F F
21642024-02-20T22:42:38.979ZMax gen: 19, Max flush: 38
21652024-02-20T22:42:38.979Z
21662024-02-20T22:42:38.979ZBegin loop 10
21672024-02-20T22:42:38.979ZDownstairs to restart: 1
21682024-02-20T22:42:39.001Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537160863Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215}
21692024-02-20T22:42:39.001Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537484922Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215}
21702024-02-20T22:42:39.001Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
21712024-02-20T22:42:39.001Z,"time":"2024-02-20T22:42:38.537517511Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215}
21722024-02-20T22:42:39.016Z{"msg":"Crucible 93462870-fdfa-4029-be58-93214f292fb6 has session id: 57612c2c-411a-420b-8113-db98ef82c452","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.53754842Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21732024-02-20T22:42:39.016Z{"msg":"Upstairs opts: Upstairs UUID: 93462870-fdfa-4029-be58-93214f292fb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537611587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21742024-02-20T22:42:39.016Z{"msg":"Crucible stats registered with UUID: 93462870-fdfa-4029-be58-93214f292fb6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537650416Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21752024-02-20T22:42:39.016Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537741093Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215}
21762024-02-20T22:42:39.016Z{"msg":"93462870-fdfa-4029-be58-93214f292fb6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537878078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21772024-02-20T22:42:39.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537903737Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21782024-02-20T22:42:39.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537918826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21792024-02-20T22:42:39.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537932596Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21802024-02-20T22:42:39.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537948095Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21812024-02-20T22:42:39.017Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537968645Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21822024-02-20T22:42:39.017Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537982484Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21832024-02-20T22:42:39.017Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.537998804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21842024-02-20T22:42:39.017Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.538026672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21852024-02-20T22:42:39.017Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.538041552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21862024-02-20T22:42:39.017Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.538230225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21872024-02-20T22:42:39.017Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.538281903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21882024-02-20T22:42:39.017Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.538312622Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21892024-02-20T22:42:39.017Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.538978138Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21902024-02-20T22:42:39.017Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539014677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21912024-02-20T22:42:39.017Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539125013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21922024-02-20T22:42:39.017Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539189901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21932024-02-20T22:42:39.017Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539235349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21942024-02-20T22:42:39.017Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539260228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21952024-02-20T22:42:39.017Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539551388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21962024-02-20T22:42:39.017Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539637035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21972024-02-20T22:42:39.017Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.539944244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21982024-02-20T22:42:39.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540130137Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
21992024-02-20T22:42:39.017Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540181625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22002024-02-20T22:42:39.017Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540205914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22012024-02-20T22:42:39.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540229334Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22022024-02-20T22:42:39.017Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540265372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22032024-02-20T22:42:39.017Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540300081Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22042024-02-20T22:42:39.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540406827Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22052024-02-20T22:42:39.018Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540447746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22062024-02-20T22:42:39.018Z{"msg":"[0]R flush_numbers[0..12]: [22, 38, 38, 37, 38, 38, 38, 38, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540475425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22072024-02-20T22:42:39.018Z{"msg":"[0]R generation[0..12]: [12, 19, 19, 18, 19, 19, 19, 19, 19, 19, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540509354Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22082024-02-20T22:42:39.018Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540536773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22092024-02-20T22:42:39.018Z{"msg":"[1]R flush_numbers[0..12]: [22, 38, 38, 37, 38, 38, 38, 38, 38, 38, 30, 29]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22102024-02-20T22:42:39.018Z,"time":"2024-02-20T22:42:38.540571201Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22112024-02-20T22:42:39.018Z{"msg":"[1]R generation[0..12]: [12, 19, 19, 18, 19, 19, 19, 19, 19, 19, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.54059939Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22122024-02-20T22:42:39.018Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540633169Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22132024-02-20T22:42:39.018Z{"msg":"[2]R flush_numbers[0..12]: [22, 38, 38, 37, 38, 38, 38, 38, 38, 38, 30, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540660188Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22142024-02-20T22:42:39.018Z{"msg":"[2]R generation[0..12]: [12, 19, 19, 18, 19, 19, 19, 19, 19, 19, 15, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540685337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22152024-02-20T22:42:39.018Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540715726Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22162024-02-20T22:42:39.018Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540740255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22172024-02-20T22:42:39.018Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540763725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22182024-02-20T22:42:39.018Z{"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540794303Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22192024-02-20T22:42:39.018Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540818272Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22202024-02-20T22:42:39.018Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540845531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22212024-02-20T22:42:39.018Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540867331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22222024-02-20T22:42:39.018Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.54089526Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22232024-02-20T22:42:39.018Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540919099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22242024-02-20T22:42:39.018Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540945568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22252024-02-20T22:42:39.018Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.540967807Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22262024-02-20T22:42:39.018Z{"msg":"93462870-fdfa-4029-be58-93214f292fb6 is now active with session: 57612c2c-411a-420b-8113-db98ef82c452","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22272024-02-20T22:42:39.018Zname":"crucible","level":30,"time":"2024-02-20T22:42:38.541000376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22282024-02-20T22:42:39.018Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.541023595Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215,"session_id":"57612c2c-411a-420b-8113-db98ef82c452"}
22292024-02-20T22:42:39.018Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:38.541045375Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215}
22302024-02-20T22:42:39.018ZLoading write count information from file "/tmp/test_repair_verify.data"
22312024-02-20T22:42:39.019ZRun Repair workload
22322024-02-20T22:42:39.019Z01/30 Write block 202 len 32768 data: 3 2 2 2 2 2 2 2
22332024-02-20T22:42:39.019Z02/30 Write block 292 len 4096 data: 2
22342024-02-20T22:42:39.019Z03/30 Flush
22352024-02-20T22:42:39.118Z04/30 Write block 154 len 32768 data: 2 2 2 2 2 2 2 2
22362024-02-20T22:42:39.139Z05/30 Read block 34 len 4096
22372024-02-20T22:42:39.139Z06/30 Read block 323 len 36864
22382024-02-20T22:42:39.140Z07/30 Flush
22392024-02-20T22:42:39.140Z08/30 Write block 144 len 4096 data: 3
22402024-02-20T22:42:39.140Z09/30 Read block 251 len 4096
22412024-02-20T22:42:39.140Z10/30 Write block 142 len 28672 data: 3 3 4 3 2 2 2
22422024-02-20T22:42:39.140Z11/30 Read block 560 len 24576
22432024-02-20T22:42:39.140Z12/30 Read block 327 len 24576
22442024-02-20T22:42:39.140Z13/30 Write block 52 len 4096 data: 2
22452024-02-20T22:42:39.152Z14/30 Write block 483 len 24576 data: 4 4 3 3 3 3
22462024-02-20T22:42:39.152Z15/30 Flush
22472024-02-20T22:42:39.152Z16/30 Write block 329 len 40960 data: 3 3 2 3 3 2 2 2 2 2
22482024-02-20T22:42:39.152Z17/30 Read block 85 len 12288
22492024-02-20T22:42:39.152Z18/30 Read block 110 len 16384
22502024-02-20T22:42:39.152Z19/30 Read block 178 len 32768
22512024-02-20T22:42:39.152Z20/30 Flush
22522024-02-20T22:42:39.607Z21/30 Write block 586 len 24576 data: 5 5 5 5 5 4
22532024-02-20T22:42:39.607Z22/30 Read block 264 len 20480
22542024-02-20T22:42:39.607Z23/30 Write block 504 len 32768 data: 2 2 2 2 2 2 2 2
22552024-02-20T22:42:39.607Z24/30 Write block 384 len 32768 data: 3 3 3 2 5 5 5 4
22562024-02-20T22:42:39.607Z25/30 Read block 454 len 20480
22572024-02-20T22:42:39.607Z26/30 Read block 270 len 20480
22582024-02-20T22:42:39.607Z27/30 Write block 405 len 40960 data: 4 5 5 4 5 5 5 5 5 5
22592024-02-20T22:42:39.607Z28/30 Read block 550 len 12288
22602024-02-20T22:42:39.607Z29/30 Write block 17 len 4096 data: 4
22612024-02-20T22:42:39.607Z30/30 Read block 236 len 20480
22622024-02-20T22:42:39.607Z----------------------------------------------------------------
22632024-02-20T22:42:39.608Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22642024-02-20T22:42:39.608ZStates: Active Active Active
22652024-02-20T22:42:39.608ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22662024-02-20T22:42:39.608Z 1 Acked 1000 Write 8 Done Done Done false
22672024-02-20T22:42:39.608Z 2 Acked 1001 Write 1 Done Done Done false
22682024-02-20T22:42:39.608Z 3 Acked 1002 Flush 0 Done Sent Done false
22692024-02-20T22:42:39.608Z 4 Acked 1003 Write 8 Done Sent Done false
22702024-02-20T22:42:39.608Z 5 Acked 1004 Read 1 Done Sent Done false
22712024-02-20T22:42:39.608Z 6 Acked 1005 Read 9 Done Sent Done false
22722024-02-20T22:42:39.608Z 7 Acked 1006 Flush 0 Done Sent Done false
22732024-02-20T22:42:39.608Z 8 Acked 1007 Write 1 Done Sent Done false
22742024-02-20T22:42:39.608Z 9 Acked 1008 Read 1 Done Sent Done false
22752024-02-20T22:42:39.609Z 10 Acked 1009 Write 7 Done Sent Done false
22762024-02-20T22:42:39.609Z 11 Acked 1010 Read 6 Done Sent Done false
22772024-02-20T22:42:39.609Z 12 Acked 1011 Read 6 Done Sent Done false
22782024-02-20T22:42:39.609Z 13 Acked 1012 Write 1 Done Sent Done false
22792024-02-20T22:42:39.609Z 14 Acked 1013 Write 6 Done Sent Done false
22802024-02-20T22:42:39.609Z 15 Acked 1014 Flush 0 Done Sent Done false
22812024-02-20T22:42:39.609Z 16 Acked 1015 Write 10 Done Sent Done false
22822024-02-20T22:42:39.609Z 17 Acked 1016 Read 3 Done Sent Done false
22832024-02-20T22:42:39.609Z 18 Acked 1017 Read 4 Done Sent Done false
22842024-02-20T22:42:39.609Z 19 Acked 1018 Read 8 Done Sent Done false
22852024-02-20T22:42:39.609Z 20 Acked 1019 Flush 0 Done Sent Done false
22862024-02-20T22:42:39.609Z 21 Acked 1020 Write 6 Done Sent Done false
22872024-02-20T22:42:39.609Z 22 Acked 1021 Read 5 Done Sent Done false
22882024-02-20T22:42:39.609Z 23 Acked 1022 Write 8 Done Sent Done false
22892024-02-20T22:42:39.609Z 24 Acked 1023 Write 8 Done Sent Done false
22902024-02-20T22:42:39.609Z 25 Acked 1024 Read 5 Done Sent Done false
22912024-02-20T22:42:39.609Z 26 Acked 1025 Read 5 Done Sent Done false
22922024-02-20T22:42:39.609Z 27 Acked 1026 Write 10 Done Sent Done false
22932024-02-20T22:42:39.609Z 28 Acked 1027 Read 3 Done Sent Done false
22942024-02-20T22:42:39.609Z 29 Acked 1028 Write 1 Done Sent Done false
22952024-02-20T22:42:39.610Z 30 Acked 1029 Read 5 Done Sent Done false
22962024-02-20T22:42:39.610Z STATES DS:0 DS:1 DS:2 TOTAL
22972024-02-20T22:42:39.610Z New 0 0 0 0
22982024-02-20T22:42:39.610Z Sent 0 28 0 28
22992024-02-20T22:42:39.610Z Done 30 2 30 62
23002024-02-20T22:42:39.610Z Skipped 0 0 0 0
23012024-02-20T22:42:39.610Z Error 0 0 0 0
23022024-02-20T22:42:39.610ZLast Flush: 1019 0 1019
23032024-02-20T22:42:39.610ZDownstairs last five completed:
23042024-02-20T22:42:39.610ZUpstairs last five completed: 30 29 28 27 26
23052024-02-20T22:42:39.610Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:38.710805389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1215}
23062024-02-20T22:42:39.610ZWrote out file "/tmp/test_repair_verify.data"
23072024-02-20T22:42:39.610Z
23082024-02-20T22:42:39.610ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23092024-02-20T22:42:39.610ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23102024-02-20T22:42:39.610Z 0 000-019 12 12 12 22 22 22 T F T <---
23112024-02-20T22:42:39.610Z 1 020-039 19 19 19 38 38 38 F F F
23122024-02-20T22:42:39.610Z 2 040-059 20 19 20 41 38 41 F F F <---
23132024-02-20T22:42:39.610Z 3 060-079 18 18 18 37 37 37 F F F
23142024-02-20T22:42:39.611Z 4 080-099 19 19 19 38 38 38 F F F
23152024-02-20T22:42:39.611Z 5 100-119 19 19 19 38 38 38 F F F
23162024-02-20T22:42:39.611Z 6 120-139 19 19 19 38 38 38 F F F
23172024-02-20T22:42:39.611Z 7 140-159 20 19 20 41 38 41 F F F <---
23182024-02-20T22:42:39.611Z 8 160-179 20 19 20 40 38 40 F F F <---
23192024-02-20T22:42:39.611Z 9 180-199 19 19 19 38 38 38 F F F
23202024-02-20T22:42:39.611Z 10 200-219 20 15 20 39 30 39 F T F <---
23212024-02-20T22:42:39.611Z 11 220-239 14 14 14 29 29 29 F F F
23222024-02-20T22:42:39.611Z 12 240-259 16 16 16 33 33 33 F F F
23232024-02-20T22:42:39.611Z 13 260-279 4 4 4 7 7 7 F F F
23242024-02-20T22:42:39.611Z 14 280-299 20 4 20 39 5 39 F T F <---
23252024-02-20T22:42:39.611Z 15 300-319 13 13 13 24 24 24 F F F
23262024-02-20T22:42:39.611Z 16 320-339 20 16 20 42 34 42 F F F <---
23272024-02-20T22:42:39.611Z 17 340-359 6 6 6 9 9 9 F F F
23282024-02-20T22:42:39.611Z 18 360-379 17 17 17 36 36 36 F F F
23292024-02-20T22:42:39.611Z 19 380-399 12 12 12 23 23 23 T F T <---
23302024-02-20T22:42:39.611Z 20 400-419 19 19 19 38 38 38 T F T <---
23312024-02-20T22:42:39.611Z 21 420-439 19 19 19 38 38 38 F F F
23322024-02-20T22:42:39.611Z 22 440-459 14 14 14 27 27 27 F F F
23332024-02-20T22:42:39.611Z 23 460-479 13 13 13 24 24 24 F F F
23342024-02-20T22:42:39.611Z 24 480-499 20 12 20 41 23 41 F F F <---
23352024-02-20T22:42:39.611Z 25 500-519 11 11 11 21 21 21 T F T <---
23362024-02-20T22:42:39.611Z 26 520-539 19 19 19 38 38 38 F F F
23372024-02-20T22:42:39.611Z 27 540-559 19 19 19 38 38 38 F F F
23382024-02-20T22:42:39.612Z 28 560-579 12 12 12 23 23 23 F F F
23392024-02-20T22:42:39.612Z 29 580-599 19 19 19 38 38 38 T F T <---
23402024-02-20T22:42:39.612ZMax gen: 20, Max flush: 42
23412024-02-20T22:42:39.612ZError: Difference in extent metadata found!
23422024-02-20T22:42:39.612ZA Difference in extent metadata is expected here
23432024-02-20T22:42:39.612ZOn loop 10
23442024-02-20T22:42:39.612Z
23452024-02-20T22:42:39.612ZVerifying data now
23462024-02-20T22:42:44.457ZNo change after verify
23472024-02-20T22:42:44.468ZLoop: 10 Downstairs dump after verify (and repair):
23482024-02-20T22:42:44.488ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23492024-02-20T22:42:44.488Z 0 000-019 21 21 21 43 43 43 F F F
23502024-02-20T22:42:44.488Z 1 020-039 19 19 19 38 38 38 F F F
23512024-02-20T22:42:44.488Z 2 040-059 20 20 20 41 41 41 F F F
23522024-02-20T22:42:44.488Z 3 060-079 18 18 18 37 37 37 F F F
23532024-02-20T22:42:44.488Z 4 080-099 19 19 19 38 38 38 F F F
23542024-02-20T22:42:44.489Z 5 100-119 19 19 19 38 38 38 F F F
23552024-02-20T22:42:44.489Z 6 120-139 19 19 19 38 38 38 F F F
23562024-02-20T22:42:44.489Z 7 140-159 20 20 20 41 41 41 F F F
23572024-02-20T22:42:44.489Z 8 160-179 20 20 20 40 40 40 F F F
23582024-02-20T22:42:44.489Z 9 180-199 19 19 19 38 38 38 F F F
23592024-02-20T22:42:44.489Z 10 200-219 20 20 20 39 39 39 F F F
23602024-02-20T22:42:44.489Z 11 220-239 14 14 14 29 29 29 F F F
23612024-02-20T22:42:44.489Z 12 240-259 16 16 16 33 33 33 F F F
23622024-02-20T22:42:44.489Z 13 260-279 4 4 4 7 7 7 F F F
23632024-02-20T22:42:44.489Z 14 280-299 20 20 20 39 39 39 F F F
23642024-02-20T22:42:44.489Z 15 300-319 13 13 13 24 24 24 F F F
23652024-02-20T22:42:44.489Z 16 320-339 20 20 20 42 42 42 F F F
23662024-02-20T22:42:44.489Z 17 340-359 6 6 6 9 9 9 F F F
23672024-02-20T22:42:44.489Z 18 360-379 17 17 17 36 36 36 F F F
23682024-02-20T22:42:44.489Z 19 380-399 21 21 21 43 43 43 F F F
23692024-02-20T22:42:44.489Z 20 400-419 21 21 21 43 43 43 F F F
23702024-02-20T22:42:44.489Z 21 420-439 19 19 19 38 38 38 F F F
23712024-02-20T22:42:44.489Z 22 440-459 14 14 14 27 27 27 F F F
23722024-02-20T22:42:44.489Z 23 460-479 13 13 13 24 24 24 F F F
23732024-02-20T22:42:44.489Z 24 480-499 20 20 20 41 41 41 F F F
23742024-02-20T22:42:44.489Z 25 500-519 21 21 21 43 43 43 F F F
23752024-02-20T22:42:44.489Z 26 520-539 19 19 19 38 38 38 F F F
23762024-02-20T22:42:44.490Z 27 540-559 19 19 19 38 38 38 F F F
23772024-02-20T22:42:44.490Z 28 560-579 12 12 12 23 23 23 F F F
23782024-02-20T22:42:44.490Z 29 580-599 21 21 21 43 43 43 F F F
23792024-02-20T22:42:44.490ZMax gen: 21, Max flush: 43
23802024-02-20T22:42:44.490Z
23812024-02-20T22:42:44.490ZBegin loop 11
23822024-02-20T22:42:44.490ZDownstairs to restart: 2
23832024-02-20T22:42:44.536Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.02895189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223}
23842024-02-20T22:42:44.536Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.02923518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223}
23852024-02-20T22:42:44.536Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029273779Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223}
23862024-02-20T22:42:44.536Z{"msg":"Crucible b922581a-8422-42fd-9c18-eba454978d48 has session id: cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029310008Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23872024-02-20T22:42:44.536Z{"msg":"Upstairs opts: Upstairs UUID: b922581a-8422-42fd-9c18-eba454978d48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
23882024-02-20T22:42:44.536Z,"time":"2024-02-20T22:42:44.029349896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23892024-02-20T22:42:44.536Z{"msg":"Crucible stats registered with UUID: b922581a-8422-42fd-9c18-eba454978d48","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029396185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23902024-02-20T22:42:44.536Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029416214Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223}
23912024-02-20T22:42:44.536Z{"msg":"b922581a-8422-42fd-9c18-eba454978d48 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029542379Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23922024-02-20T22:42:44.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029569208Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23932024-02-20T22:42:44.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029584828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23942024-02-20T22:42:44.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029606767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23952024-02-20T22:42:44.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029629846Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23962024-02-20T22:42:44.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029649935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23972024-02-20T22:42:44.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029686184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23982024-02-20T22:42:44.536Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029706503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
23992024-02-20T22:42:44.536Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029771931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24002024-02-20T22:42:44.536Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029859398Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24012024-02-20T22:42:44.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029892967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24022024-02-20T22:42:44.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029918696Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24032024-02-20T22:42:44.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.029956454Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24042024-02-20T22:42:44.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.030617571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24052024-02-20T22:42:44.537Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.030671469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24062024-02-20T22:42:44.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.030704978Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24072024-02-20T22:42:44.537Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.030746266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24082024-02-20T22:42:44.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.030858402Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24092024-02-20T22:42:44.537Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.030899241Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24102024-02-20T22:42:44.537Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.03120053Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24112024-02-20T22:42:44.537Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031239428Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24122024-02-20T22:42:44.537Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031257408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24132024-02-20T22:42:44.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031781079Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24142024-02-20T22:42:44.537Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031828187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24152024-02-20T22:42:44.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031850857Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24162024-02-20T22:42:44.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031874536Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24172024-02-20T22:42:44.537Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031889595Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24182024-02-20T22:42:44.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031904225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24192024-02-20T22:42:44.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031916874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24202024-02-20T22:42:44.537Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031937814Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24212024-02-20T22:42:44.538Z{"msg":"[0]R flush_numbers[0..12]: [43, 38, 41, 37, 38, 38, 38, 41, 40, 38, 39, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031952133Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24222024-02-20T22:42:44.538Z{"msg":"[0]R generation[0..12]: [21, 19, 20, 18, 19, 19, 19, 20, 20, 19, 20, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031972832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24232024-02-20T22:42:44.538Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.031987942Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO
24242024-02-20T22:42:44.538Z"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24252024-02-20T22:42:44.538Z{"msg":"[1]R flush_numbers[0..12]: [43, 38, 41, 37, 38, 38, 38, 41, 40, 38, 39, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032019691Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24262024-02-20T22:42:44.538Z{"msg":"[1]R generation[0..12]: [21, 19, 20, 18, 19, 19, 19, 20, 20, 19, 20, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.03204199Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24272024-02-20T22:42:44.538Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032067569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24282024-02-20T22:42:44.538Z{"msg":"[2]R flush_numbers[0..12]: [43, 38, 41, 37, 38, 38, 38, 41, 40, 38, 39, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032092768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24292024-02-20T22:42:44.538Z{"msg":"[2]R generation[0..12]: [21, 19, 20, 18, 19, 19, 19, 20, 20, 19, 20, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032107217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24302024-02-20T22:42:44.538Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032122877Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24312024-02-20T22:42:44.538Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032136346Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24322024-02-20T22:42:44.538Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032157696Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24332024-02-20T22:42:44.538Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032171035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24342024-02-20T22:42:44.538Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032185925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24352024-02-20T22:42:44.538Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032199584Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24362024-02-20T22:42:44.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032219273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24372024-02-20T22:42:44.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032233083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24382024-02-20T22:42:44.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032247032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24392024-02-20T22:42:44.538Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032260362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24402024-02-20T22:42:44.538Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032272751Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24412024-02-20T22:42:44.539Z{"msg":"b922581a-8422-42fd-9c18-eba454978d48 is now active with session: cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032292601Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24422024-02-20T22:42:44.539Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.03230543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223,"session_id":"cc9a1def-e59d-4ba4-a1fc-8723f4cdab2f"}
24432024-02-20T22:42:44.539Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:44.032326409Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223}
24442024-02-20T22:42:44.539ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24452024-02-20T22:42:44.539ZLoading write count information from file "/tmp/test_repair_verify.data"
24462024-02-20T22:42:44.539ZRun Repair workload
24472024-02-20T22:42:44.539Z01/30 Write block 462 len 8192 data: 5 5
24482024-02-20T22:42:44.539Z02/30 Write block 496 len 8192 data: 3 3
24492024-02-20T22:42:44.539Z03/30 Write block 136 len 20480 data: 4 4 5 3 3
24502024-02-20T22:42:44.539Z04/30 Read block 61 len 16384
24512024-02-20T22:42:44.539Z05/30 Read block 78 len 16384
24522024-02-20T22:42:44.539Z06/30 Read block 107 len 32768
24532024-02-20T22:42:44.572Z07/30 Read block 213 len 36864
24542024-02-20T22:42:44.572Z08/30 Read block 373 len 40960
24552024-02-20T22:42:44.572Z09/30 Write block 333 len 4096 data: 4
24562024-02-20T22:42:44.572Z10/30 Read block 340 len 8192
24572024-02-20T22:42:44.572Z11/30 Flush
24582024-02-20T22:42:44.572Z12/30 Write block 382 len 16384 data: 3 3 4 4
24592024-02-20T22:42:44.572Z13/30 Read block 29 len 20480
24602024-02-20T22:42:44.572Z14/30 Write block 370 len 40960 data: 3 4 5 3 3 3 3 4 3 2
24612024-02-20T22:42:44.572Z15/30 Write block 309 len 36864 data: 3 3 3 3 3 3 3 2 2
24622024-02-20T22:42:44.572Z16/30 Write block 179 len 24576 data: 5 6 6 5 4 4
24632024-02-20T22:42:44.572Z17/30 Read block 99 len 8192
24642024-02-20T22:42:44.572Z18/30 Write block 25 len 32768 data: 4 4 4 5 4 5 5 5
24652024-02-20T22:42:44.572Z19/30 Read block 331 len 40960
24662024-02-20T22:42:44.572Z20/30 Read block 73 len 28672
24672024-02-20T22:42:44.572Z21/30 Read block 161 len 12288
24682024-02-20T22:42:44.572Z22/30 Write block 364 len 32768 data: 6 6 5 4 4 3 4 5
24692024-02-20T22:42:44.572Z23/30 Write block 549 len 40960 data: 3 4 3 3 2 2 2 2 2 2
24702024-02-20T22:42:44.573Z24/30 Read block 525 len 20480
24712024-02-20T22:42:44.573Z25/30 Read block 120 len 36864
24722024-02-20T22:42:44.573Z26/30 Read block 515 len 24576
24732024-02-20T22:42:44.573Z27/30 Read block 20 len 8192
24742024-02-20T22:42:44.573Z28/30 Read block 398 len 28672
24752024-02-20T22:42:44.573Z29/30 Read block 188 len 20480
24762024-02-20T22:42:44.573Z30/30 Write block 150 len 24576 data: 5 5 4 3 3 3
24772024-02-20T22:42:44.573Z----------------------------------------------------------------
24782024-02-20T22:42:44.573Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24792024-02-20T22:42:44.573ZStates: Active Active Active
24802024-02-20T22:42:44.573ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24812024-02-20T22:42:44.573Z 1 Acked 1000 Write 2 Done Done Done false
24822024-02-20T22:42:44.573Z 2 Acked 1001 Write 2 Done Done Done false
24832024-02-20T22:42:44.573Z 3 Acked 1002 Write 5 Done Done Sent false
24842024-02-20T22:42:44.573Z 4 Acked 1003 Read 4 Done Done Sent false
24852024-02-20T22:42:44.573Z 5 Acked 1004 Read 4 Done Done Sent false
24862024-02-20T22:42:44.573Z 6 Acked 1005 Read 8 Done Done Sent false
24872024-02-20T22:42:44.573Z 7 Acked 1006 Read 9 Done Done Sent false
24882024-02-20T22:42:44.573Z 8 Acked 1007 Read 10 Done Done Sent false
24892024-02-20T22:42:44.573Z 9 Acked 1008 Write 1 Done Done Sent false
24902024-02-20T22:42:44.573Z 10 Acked 1009 Read 2 Done Done Sent false
24912024-02-20T22:42:44.573Z 11 Acked 1010 Flush 0 Done Done Sent false
24922024-02-20T22:42:44.573Z 12 Acked 1011 Write 4 Done Done Sent false
24932024-02-20T22:42:44.573Z 13 Acked 1012 Read 5 Done Done Sent false
24942024-02-20T22:42:44.574Z 14 Acked 1013 Write 10 Done Done Sent false
24952024-02-20T22:42:44.574Z 15 Acked 1014 Write 9 Done Done Sent false
24962024-02-20T22:42:44.574Z 16 Acked 1015 Write 6 Done Done Sent false
24972024-02-20T22:42:44.574Z 17 Acked 1016 Read 2 Done Done Sent false
24982024-02-20T22:42:44.574Z 18 Acked 1017 Write 8 Done Done Sent false
24992024-02-20T22:42:44.574Z 19 Acked 1018 Read 10 Done Done Sent false
25002024-02-20T22:42:44.574Z 20 Acked 1019 Read 7 Done Done Sent false
25012024-02-20T22:42:44.574Z 21 Acked 1020 Read 3 Done Done Sent false
25022024-02-20T22:42:44.574Z 22 Acked 1021 Write 8 Done Done Sent false
25032024-02-20T22:42:44.574Z 23 Acked 1022 Write 10 Done Done Sent false
25042024-02-20T22:42:44.574Z 24 Acked 1023 Read 5 Done Done Sent false
25052024-02-20T22:42:44.574Z 25 Acked 1024 Read 9 Done Done Sent false
25062024-02-20T22:42:44.574Z 26 Acked 1025 Read 6 Done Done Sent false
25072024-02-20T22:42:44.574Z 27 Acked 1026 Read 2 Done Done Sent false
25082024-02-20T22:42:44.574Z 28 Acked 1027 Read 7 Done Done Sent false
25092024-02-20T22:42:44.575Z 29 Acked 1028 Read 5 Done Done Sent false
25102024-02-20T22:42:44.575Z 30 Acked 1029 Write 6 Sent Sent Sent false
25112024-02-20T22:42:44.575Z STATES DS:0 DS:1 DS:2 TOTAL
25122024-02-20T22:42:44.575Z New 0 0 0 0
25132024-02-20T22:42:44.575Z Sent 1 1 28 30
25142024-02-20T22:42:44.575Z Done 29 29 2 60
25152024-02-20T22:42:44.575Z Skipped 0 0 0 0
25162024-02-20T22:42:44.575Z Error 0 0 0 0
25172024-02-20T22:42:44.575ZLast Flush: 1010 1010 0
25182024-02-20T22:42:44.575ZDownstairs last five completed:
25192024-02-20T22:42:44.575ZUpstairs last five completed: 30 29 28 27 26
25202024-02-20T22:42:44.575Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:44.061169228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1223}
25212024-02-20T22:42:44.575ZWrote out file "/tmp/test_repair_verify.data"
25222024-02-20T22:42:44.575Z
25232024-02-20T22:42:44.575ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25242024-02-20T22:42:44.575ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25252024-02-20T22:42:44.575Z 0 000-019 21 21 21 43 43 43 F F F
25262024-02-20T22:42:44.575Z 1 020-039 19 19 19 38 38 38 T T F <---
25272024-02-20T22:42:44.575Z 2 040-059 20 20 20 41 41 41 F F F
25282024-02-20T22:42:44.575Z 3 060-079 18 18 18 37 37 37 F F F
25292024-02-20T22:42:44.575Z 4 080-099 19 19 19 38 38 38 F F F
25302024-02-20T22:42:44.575Z 5 100-119 19 19 19 38 38 38 F F F
25312024-02-20T22:42:44.575Z 6 120-139 22 22 19 44 44 38 F F F <---
25322024-02-20T22:42:44.575Z 7 140-159 22 22 20 44 44 41 T T F <---
25332024-02-20T22:42:44.575Z 8 160-179 20 20 20 40 40 40 T T F <---
25342024-02-20T22:42:44.576Z 9 180-199 19 19 19 38 38 38 T T F <---
25352024-02-20T22:42:44.576Z 10 200-219 20 20 20 39 39 39 F F F
25362024-02-20T22:42:44.576Z 11 220-239 14 14 14 29 29 29 F F F
25372024-02-20T22:42:44.576Z 12 240-259 16 16 16 33 33 33 F F F
25382024-02-20T22:42:44.576Z 13 260-279 4 4 4 7 7 7 F F F
25392024-02-20T22:42:44.576Z 14 280-299 20 20 20 39 39 39 F F F
25402024-02-20T22:42:44.576Z 15 300-319 13 13 13 24 24 24 T T F <---
25412024-02-20T22:42:44.576Z 16 320-339 22 22 20 44 44 42 F F F <---
25422024-02-20T22:42:44.576Z 17 340-359 6 6 6 9 9 9 F F F
25432024-02-20T22:42:44.576Z 18 360-379 17 17 17 36 36 36 T T F <---
25442024-02-20T22:42:44.576Z 19 380-399 21 21 21 43 43 43 T T F <---
25452024-02-20T22:42:44.576Z 20 400-419 21 21 21 43 43 43 F F F
25462024-02-20T22:42:44.576Z 21 420-439 19 19 19 38 38 38 F F F
25472024-02-20T22:42:44.576Z 22 440-459 14 14 14 27 27 27 F F F
25482024-02-20T22:42:44.576Z 23 460-479 22 22 13 44 44 24 F F T <---
25492024-02-20T22:42:44.576Z 24 480-499 22 22 20 44 44 41 F F T <---
25502024-02-20T22:42:44.576Z 25 500-519 21 21 21 43 43 43 F F F
25512024-02-20T22:42:44.576Z 26 520-539 19 19 19 38 38 38 F F F
25522024-02-20T22:42:44.576Z 27 540-559 19 19 19 38 38 38 T T F <---
25532024-02-20T22:42:44.595Z 28 560-579 12 12 12 23 23 23 F F F
25542024-02-20T22:42:44.595Z 29 580-599 21 21 21 43 43 43 F F F
25552024-02-20T22:42:44.595ZMax gen: 22, Max flush: 44
25562024-02-20T22:42:44.595ZError: Difference in extent metadata found!
25572024-02-20T22:42:44.595ZA Difference in extent metadata is expected here
25582024-02-20T22:42:44.595ZOn loop 11
25592024-02-20T22:42:44.595Z
25602024-02-20T22:42:44.595ZVerifying data now
25612024-02-20T22:42:50.844ZNo change after verify
25622024-02-20T22:42:50.844ZLoop: 11 Downstairs dump after verify (and repair):
25632024-02-20T22:42:50.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25642024-02-20T22:42:50.856Z 0 000-019 21 21 21 43 43 43 F F F
25652024-02-20T22:42:50.856Z 1 020-039 23 23 23 45 45 45 F F F
25662024-02-20T22:42:50.856Z 2 040-059 20 20 20 41 41 41 F F F
25672024-02-20T22:42:50.856Z 3 060-079 18 18 18 37 37 37 F F F
25682024-02-20T22:42:50.856Z 4 080-099 19 19 19 38 38 38 F F F
25692024-02-20T22:42:50.874Z 5 100-119 19 19 19 38 38 38 F F F
25702024-02-20T22:42:50.874Z 6 120-139 22 22 22 44 44 44 F F F
25712024-02-20T22:42:50.874Z 7 140-159 23 23 23 45 45 45 F F F
25722024-02-20T22:42:50.874Z 8 160-179 23 23 23 45 45 45 F F F
25732024-02-20T22:42:50.874Z 9 180-199 23 23 23 45 45 45 F F F
25742024-02-20T22:42:50.874Z 10 200-219 20 20 20 39 39 39 F F F
25752024-02-20T22:42:50.874Z 11 220-239 14 14 14 29 29 29 F F F
25762024-02-20T22:42:50.874Z 12 240-259 16 16 16 33 33 33 F F F
25772024-02-20T22:42:50.874Z 13 260-279 4 4 4 7 7 7 F F F
25782024-02-20T22:42:50.874Z 14 280-299 20 20 20 39 39 39 F F F
25792024-02-20T22:42:50.874Z 15 300-319 23 23 23 45 45 45 F F F
25802024-02-20T22:42:50.874Z 16 320-339 22 22 22 44 44 44 F F F
25812024-02-20T22:42:50.874Z 17 340-359 6 6 6 9 9 9 F F F
25822024-02-20T22:42:50.874Z 18 360-379 23 23 23 45 45 45 F F F
25832024-02-20T22:42:50.874Z 19 380-399 23 23 23 45 45 45 F F F
25842024-02-20T22:42:50.874Z 20 400-419 21 21 21 43 43 43 F F F
25852024-02-20T22:42:50.875Z 21 420-439 19 19 19 38 38 38 F F F
25862024-02-20T22:42:50.875Z 22 440-459 14 14 14 27 27 27 F F F
25872024-02-20T22:42:50.875Z 23 460-479 22 22 22 44 44 44 F F F
25882024-02-20T22:42:50.875Z 24 480-499 22 22 22 44 44 44 F F F
25892024-02-20T22:42:50.875Z 25 500-519 21 21 21 43 43 43 F F F
25902024-02-20T22:42:50.875Z 26 520-539 19 19 19 38 38 38 F F F
25912024-02-20T22:42:50.875Z 27 540-559 23 23 23 45 45 45 F F F
25922024-02-20T22:42:50.875Z 28 560-579 12 12 12 23 23 23 F F F
25932024-02-20T22:42:50.875Z 29 580-599 21 21 21 43 43 43 F F F
25942024-02-20T22:42:50.875ZMax gen: 23, Max flush: 45
25952024-02-20T22:42:50.875Z
25962024-02-20T22:42:50.875ZBegin loop 12
25972024-02-20T22:42:50.875ZDownstairs to restart: 0
25982024-02-20T22:42:50.909Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412213329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231}
25992024-02-20T22:42:50.909Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412536837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231}
26002024-02-20T22:42:50.909Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
26012024-02-20T22:42:50.909Z,"time":"2024-02-20T22:42:50.412581505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231}
26022024-02-20T22:42:50.909Z{"msg":"Crucible c6ee5a16-e952-4bb7-a952-1590300c8423 has session id: d028579f-85d2-4a2e-a931-82a9b9ce1f26","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412612114Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26032024-02-20T22:42:50.909Z{"msg":"Upstairs opts: Upstairs UUID: c6ee5a16-e952-4bb7-a952-1590300c8423, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412650183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26042024-02-20T22:42:50.909Z{"msg":"Crucible stats registered with UUID: c6ee5a16-e952-4bb7-a952-1590300c8423","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412679892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26052024-02-20T22:42:50.909Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412696571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231}
26062024-02-20T22:42:50.909Z{"msg":"c6ee5a16-e952-4bb7-a952-1590300c8423 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412807997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26072024-02-20T22:42:50.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412855446Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26082024-02-20T22:42:50.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412885774Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26092024-02-20T22:42:50.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412922943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26102024-02-20T22:42:50.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412958742Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26112024-02-20T22:42:50.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.412995281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26122024-02-20T22:42:50.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.4130219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26132024-02-20T22:42:50.909Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.413101697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26142024-02-20T22:42:50.909Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.413157695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26152024-02-20T22:42:50.909Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.413189994Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26162024-02-20T22:42:50.909Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41329028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26172024-02-20T22:42:50.909Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.413339848Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26182024-02-20T22:42:50.909Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.413366667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26192024-02-20T22:42:50.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.414035803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26202024-02-20T22:42:50.910Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.414072552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26212024-02-20T22:42:50.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.414087811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26222024-02-20T22:42:50.910Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.414101541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26232024-02-20T22:42:50.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41412279Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26242024-02-20T22:42:50.910Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41413694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26252024-02-20T22:42:50.910Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.414630842Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26262024-02-20T22:42:50.910Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.414668221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26272024-02-20T22:42:50.910Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41468425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26282024-02-20T22:42:50.910Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415159723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26292024-02-20T22:42:50.910Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415197362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26302024-02-20T22:42:50.910Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415212541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26312024-02-20T22:42:50.910Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415225361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26322024-02-20T22:42:50.910Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41523971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26332024-02-20T22:42:50.910Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41526236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26342024-02-20T22:42:50.910Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415275569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26352024-02-20T22:42:50.910Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415296038Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26362024-02-20T22:42:50.910Z{"msg":"[0]R flush_numbers[0..12]: [43, 45, 41, 37, 38, 38, 44, 45, 45, 45, 39, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415310158Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26372024-02-20T22:42:50.910Z{"msg":"[0]R generation[0..12]: [21, 23, 20, 18, 19, 19, 22, 23, 23, 23, 20, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415330507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26382024-02-20T22:42:50.910Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415345046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26392024-02-20T22:42:50.910Z{"msg":"[1]R flush_numbers[0..12]: [43, 45, 41, 37, 38, 38, 44, 45, 45, 45, 39, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415365756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26402024-02-20T22:42:50.910Z{"msg":"[1]R generation[0..12]: [21, 23, 20, 18, 19, 19, 22, 23, 23, 23, 20, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415379695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26412024-02-20T22:42:50.910ZWait for a query_work_queue command to finish before sending IO
26422024-02-20T22:42:50.910Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415406294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26432024-02-20T22:42:50.911Z{"msg":"[2]R flush_numbers[0..12]: [43, 45, 41, 37, 38, 38, 44, 45, 45, 45, 39, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415427294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26442024-02-20T22:42:50.911Z{"msg":"[2]R generation[0..12]: [21, 23, 20, 18, 19, 19, 22, 23, 23, 23, 20, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415441423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26452024-02-20T22:42:50.911Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415462022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26462024-02-20T22:42:50.911Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415475762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26472024-02-20T22:42:50.911Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415495761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26482024-02-20T22:42:50.911Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415509051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26492024-02-20T22:42:50.911Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.41552886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26502024-02-20T22:42:50.911Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415542679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26512024-02-20T22:42:50.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415561689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26522024-02-20T22:42:50.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415575378Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26532024-02-20T22:42:50.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415588788Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26542024-02-20T22:42:50.911Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415608687Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26552024-02-20T22:42:50.911Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415621336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26562024-02-20T22:42:50.911Z{"msg":"c6ee5a16-e952-4bb7-a952-1590300c8423 is now active with session: d028579f-85d2-4a2e-a931-82a9b9ce1f26","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415640656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26572024-02-20T22:42:50.911Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415653015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231,"session_id":"d028579f-85d2-4a2e-a931-82a9b9ce1f26"}
26582024-02-20T22:42:50.911Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:50.415672215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231}
26592024-02-20T22:42:50.911ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26602024-02-20T22:42:50.911ZLoading write count information from file "/tmp/test_repair_verify.data"
26612024-02-20T22:42:50.911ZRun Repair workload
26622024-02-20T22:42:50.911Z01/30 Write block 365 len 32768 data: 7 6 5 5 4 5 6 6
26632024-02-20T22:42:50.911Z02/30 Flush
26642024-02-20T22:42:50.918Z03/30 Write block 57 len 12288 data: 2 3 3
26652024-02-20T22:42:50.918Z04/30 Flush
26662024-02-20T22:42:50.918Z05/30 Write block 117 len 16384 data: 6 3 3 3
26672024-02-20T22:42:50.919Z06/30 Read block 256 len 20480
26682024-02-20T22:42:50.919Z07/30 Write block 168 len 24576 data: 4 4 4 3 3 3
26692024-02-20T22:42:50.919Z08/30 Read block 116 len 40960
26702024-02-20T22:42:50.919Z09/30 Read block 147 len 32768
26712024-02-20T22:42:50.919Z10/30 Read block 278 len 4096
26722024-02-20T22:42:50.919Z11/30 Write block 10 len 4096 data: 2
26732024-02-20T22:42:50.919Z12/30 Flush
26742024-02-20T22:42:50.926Z13/30 Write block 74 len 32768 data: 3 2 2 2 2 2 2 3
26752024-02-20T22:42:50.926Z14/30 Read block 63 len 12288
26762024-02-20T22:42:50.926Z15/30 Write block 207 len 24576 data: 3 3 3 2 2 2
26772024-02-20T22:42:50.926Z16/30 Read block 519 len 32768
26782024-02-20T22:42:50.926Z17/30 Write block 264 len 32768 data: 3 3 3 3 3 3 3 3
26792024-02-20T22:42:50.926Z18/30 Read block 457 len 36864
26802024-02-20T22:42:50.926Z19/30 Write block 582 len 8192 data: 4 4
26812024-02-20T22:42:50.933Z20/30 Read block 158 len 8192
26822024-02-20T22:42:50.933Z21/30 Write block 358 len 36864 data: 2 3 4 4 4 6 7 8 7
26832024-02-20T22:42:50.933Z22/30 Read block 10 len 36864
26842024-02-20T22:42:50.933Z23/30 Read block 302 len 4096
26852024-02-20T22:42:50.933Z24/30 Write block 331 len 40960 data: 3 4 5 3 3 3 3 3 3 2
26862024-02-20T22:42:50.933Z25/30 Read block 47 len 28672
26872024-02-20T22:42:50.933Z26/30 Write block 90 len 8192 data: 2 2
26882024-02-20T22:42:50.933Z27/30 Read block 272 len 12288
26892024-02-20T22:42:50.944Z28/30 Write block 409 len 36864 data: 6 6 6 6 6 6 5 5 3
26902024-02-20T22:42:50.944Z29/30 Write block 17 len 24576 data: 5 3 3 3 3 3
26912024-02-20T22:42:50.944Z30/30 Write block 199 len 32768 data: 4 3 3 4 3 3 3 3
26922024-02-20T22:42:50.944Z----------------------------------------------------------------
26932024-02-20T22:42:50.944Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:28
26942024-02-20T22:42:50.944ZStates: Active Active Active
26952024-02-20T22:42:50.944ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26962024-02-20T22:42:50.944Z 3 Acked 1002 Write 3 Done Done Done false
26972024-02-20T22:42:50.944Z 4 Acked 1003 Flush 0 Sent Done Done false
26982024-02-20T22:42:50.944Z 5 Acked 1004 Write 4 Sent Done Done false
26992024-02-20T22:42:50.944Z 6 Acked 1005 Read 5 Sent Done Done false
27002024-02-20T22:42:50.944Z 7 Acked 1006 Write 6 Sent Done Done false
27012024-02-20T22:42:50.944Z 8 Acked 1007 Read 10 Sent Done Done false
27022024-02-20T22:42:50.944Z 9 Acked 1008 Read 8 Sent Done Done false
27032024-02-20T22:42:50.944Z 10 Acked 1009 Read 1 Sent Done Done false
27042024-02-20T22:42:50.944Z 11 Acked 1010 Write 1 Sent Done Done false
27052024-02-20T22:42:50.945Z 12 Acked 1011 Flush 0 Sent Done Done false
27062024-02-20T22:42:50.945Z 13 Acked 1012 Write 8 Sent Done Done false
27072024-02-20T22:42:50.945Z 14 Acked 1013 Read 3 Sent Done Done false
27082024-02-20T22:42:50.945Z 15 Acked 1014 Write 6 Sent Done Done false
27092024-02-20T22:42:50.945Z 16 Acked 1015 Read 8 Sent Done Done false
27102024-02-20T22:42:50.945Z 17 Acked 1016 Write 8 Sent Done Done false
27112024-02-20T22:42:50.945Z 18 Acked 1017 Read 9 Sent Done Done false
27122024-02-20T22:42:50.945Z 19 Acked 1018 Write 2 Sent Done Done false
27132024-02-20T22:42:50.945Z 20 Acked 1019 Read 2 Sent Done Done false
27142024-02-20T22:42:50.945Z 21 Acked 1020 Write 9 Sent Done Done false
27152024-02-20T22:42:50.945Z 22 Acked 1021 Read 9 Sent Done Done false
27162024-02-20T22:42:50.945Z 23 Acked 1022 Read 1 Sent Done Done false
27172024-02-20T22:42:50.945Z 24 Acked 1023 Write 10 Sent Done Done false
27182024-02-20T22:42:50.945Z 25 Acked 1024 Read 7 Sent Done Done false
27192024-02-20T22:42:50.945Z 26 Acked 1025 Write 2 Sent Done Done false
27202024-02-20T22:42:50.945Z 27 Acked 1026 Read 3 Sent Done Done false
27212024-02-20T22:42:50.945Z 28 Acked 1027 Write 9 Sent Done Done false
27222024-02-20T22:42:50.945Z 29 Acked 1028 Write 6 Sent Done Done false
27232024-02-20T22:42:50.946Z 30 Acked 1029 Write 8 Sent Sent Sent false
27242024-02-20T22:42:50.946Z STATES DS:0 DS:1 DS:2 TOTAL
27252024-02-20T22:42:50.946Z New 0 0 0 0
27262024-02-20T22:42:50.946Z Sent 27 1 1 29
27272024-02-20T22:42:50.946Z Done 1 27 27 55
27282024-02-20T22:42:50.946Z Skipped 0 0 0 0
27292024-02-20T22:42:50.946Z Error 0 0 0 0
27302024-02-20T22:42:50.946ZLast Flush: 1001 1011 1011
27312024-02-20T22:42:50.946ZDownstairs last five completed: 1001 1000
27322024-02-20T22:42:50.946ZUpstairs last five completed: 30 29 28 27 26
27332024-02-20T22:42:50.946Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:50.473221755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1231}
27342024-02-20T22:42:50.946ZWrote out file "/tmp/test_repair_verify.data"
27352024-02-20T22:42:50.947Z
27362024-02-20T22:42:50.960ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27372024-02-20T22:42:50.975ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27382024-02-20T22:42:50.975Z 0 000-019 21 24 24 43 48 48 F T T <---
27392024-02-20T22:42:50.975Z 1 020-039 23 23 23 45 45 45 F T T <---
27402024-02-20T22:42:50.975Z 2 040-059 20 24 24 41 47 47 T F F <---
27412024-02-20T22:42:50.975Z 3 060-079 18 18 18 37 37 37 F T T <---
27422024-02-20T22:42:50.975Z 4 080-099 19 19 19 38 38 38 F T T <---
27432024-02-20T22:42:50.975Z 5 100-119 19 24 24 38 48 48 F F F <---
27442024-02-20T22:42:50.975Z 6 120-139 22 24 24 44 48 48 F F F <---
27452024-02-20T22:42:50.975Z 7 140-159 23 23 23 45 45 45 F F F
27462024-02-20T22:42:50.975Z 8 160-179 23 24 24 45 48 48 F F F <---
27472024-02-20T22:42:50.975Z 9 180-199 23 23 23 45 45 45 F T T <---
27482024-02-20T22:42:50.975Z 10 200-219 20 20 20 39 39 39 F T T <---
27492024-02-20T22:42:50.975Z 11 220-239 14 14 14 29 29 29 F F F
27502024-02-20T22:42:50.976Z 12 240-259 16 16 16 33 33 33 F F F
27512024-02-20T22:42:50.976Z 13 260-279 4 4 4 7 7 7 F T T <---
27522024-02-20T22:42:50.976Z 14 280-299 20 20 20 39 39 39 F F F
27532024-02-20T22:42:50.976Z 15 300-319 23 23 23 45 45 45 F F F
27542024-02-20T22:42:50.976Z 16 320-339 22 22 22 44 44 44 F T T <---
27552024-02-20T22:42:50.976Z 17 340-359 6 6 6 9 9 9 F T T <---
27562024-02-20T22:42:50.976Z 18 360-379 24 24 24 46 46 46 F T T <---
27572024-02-20T22:42:50.976Z 19 380-399 23 23 23 45 45 45 F F F
27582024-02-20T22:42:50.976Z 20 400-419 21 21 21 43 43 43 F T T <---
27592024-02-20T22:42:50.976Z 21 420-439 19 19 19 38 38 38 F F F
27602024-02-20T22:42:50.976Z 22 440-459 14 14 14 27 27 27 F F F
27612024-02-20T22:42:50.976Z 23 460-479 22 22 22 44 44 44 F F F
27622024-02-20T22:42:50.976Z 24 480-499 22 22 22 44 44 44 F F F
27632024-02-20T22:42:50.976Z 25 500-519 21 21 21 43 43 43 F F F
27642024-02-20T22:42:50.976Z 26 520-539 19 19 19 38 38 38 F F F
27652024-02-20T22:42:50.976Z 27 540-559 23 23 23 45 45 45 F F F
27662024-02-20T22:42:50.976Z 28 560-579 12 12 12 23 23 23 F F F
27672024-02-20T22:42:50.976Z 29 580-599 21 21 21 43 43 43 F T T <---
27682024-02-20T22:42:50.976ZMax gen: 24, Max flush: 48
27692024-02-20T22:42:50.976ZError: Difference in extent metadata found!
27702024-02-20T22:42:50.976ZA Difference in extent metadata is expected here
27712024-02-20T22:42:50.976ZOn loop 12
27722024-02-20T22:42:50.976Z
27732024-02-20T22:42:50.976ZVerifying data now
27742024-02-20T22:42:56.654ZNo change after verify
27752024-02-20T22:42:56.654ZLoop: 12 Downstairs dump after verify (and repair):
27762024-02-20T22:42:56.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27772024-02-20T22:42:56.668Z 0 000-019 25 25 25 49 49 49 F F F
27782024-02-20T22:42:56.668Z 1 020-039 25 25 25 49 49 49 F F F
27792024-02-20T22:42:56.668Z 2 040-059 24 24 24 47 47 47 F F F
27802024-02-20T22:42:56.668Z 3 060-079 25 25 25 49 49 49 F F F
27812024-02-20T22:42:56.668Z 4 080-099 25 25 25 49 49 49 F F F
27822024-02-20T22:42:56.668Z 5 100-119 24 24 24 48 48 48 F F F
27832024-02-20T22:42:56.668Z 6 120-139 24 24 24 48 48 48 F F F
27842024-02-20T22:42:56.668Z 7 140-159 23 23 23 45 45 45 F F F
27852024-02-20T22:42:56.668Z 8 160-179 24 24 24 48 48 48 F F F
27862024-02-20T22:42:56.668Z 9 180-199 25 25 25 49 49 49 F F F
27872024-02-20T22:42:56.668Z 10 200-219 25 25 25 49 49 49 F F F
27882024-02-20T22:42:56.668Z 11 220-239 14 14 14 29 29 29 F F F
27892024-02-20T22:42:56.668Z 12 240-259 16 16 16 33 33 33 F F F
27902024-02-20T22:42:56.673Z 13 260-279 25 25 25 49 49 49 F F F
27912024-02-20T22:42:56.673Z 14 280-299 20 20 20 39 39 39 F F F
27922024-02-20T22:42:56.673Z 15 300-319 23 23 23 45 45 45 F F F
27932024-02-20T22:42:56.673Z 16 320-339 25 25 25 49 49 49 F F F
27942024-02-20T22:42:56.673Z 17 340-359 25 25 25 49 49 49 F F F
27952024-02-20T22:42:56.673Z 18 360-379 25 25 25 49 49 49 F F F
27962024-02-20T22:42:56.673Z 19 380-399 23 23 23 45 45 45 F F F
27972024-02-20T22:42:56.673Z 20 400-419 25 25 25 49 49 49 F F F
27982024-02-20T22:42:56.674Z 21 420-439 19 19 19 38 38 38 F F F
27992024-02-20T22:42:56.674Z 22 440-459 14 14 14 27 27 27 F F F
28002024-02-20T22:42:56.674Z 23 460-479 22 22 22 44 44 44 F F F
28012024-02-20T22:42:56.674Z 24 480-499 22 22 22 44 44 44 F F F
28022024-02-20T22:42:56.674Z 25 500-519 21 21 21 43 43 43 F F F
28032024-02-20T22:42:56.674Z 26 520-539 19 19 19 38 38 38 F F F
28042024-02-20T22:42:56.674Z 27 540-559 23 23 23 45 45 45 F F F
28052024-02-20T22:42:56.674Z 28 560-579 12 12 12 23 23 23 F F F
28062024-02-20T22:42:56.674Z 29 580-599 25 25 25 49 49 49 F F F
28072024-02-20T22:42:56.674ZMax gen: 25, Max flush: 49
28082024-02-20T22:42:56.674Z
28092024-02-20T22:42:56.674ZBegin loop 13
28102024-02-20T22:42:56.674ZDownstairs to restart: 0
28112024-02-20T22:42:56.690Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227183391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239}
28122024-02-20T22:42:56.719Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
28132024-02-20T22:42:56.719Z,"v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227547868Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239}
28142024-02-20T22:42:56.719Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227589187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239}
28152024-02-20T22:42:56.719Z{"msg":"Crucible 7db37de8-c097-4717-93bd-113eff538192 has session id: 13e2b267-c705-4c4e-95e0-c0c01f2bf2c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227607796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28162024-02-20T22:42:56.719Z{"msg":"Upstairs opts: Upstairs UUID: 7db37de8-c097-4717-93bd-113eff538192, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227644985Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28172024-02-20T22:42:56.719Z{"msg":"Crucible stats registered with UUID: 7db37de8-c097-4717-93bd-113eff538192","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227662914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28182024-02-20T22:42:56.719Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227687813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239}
28192024-02-20T22:42:56.719Z{"msg":"7db37de8-c097-4717-93bd-113eff538192 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227710193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28202024-02-20T22:42:56.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227726492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28212024-02-20T22:42:56.719Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227752061Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28222024-02-20T22:42:56.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.2277827Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28232024-02-20T22:42:56.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227828348Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28242024-02-20T22:42:56.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227861587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28252024-02-20T22:42:56.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227889666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28262024-02-20T22:42:56.720Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227910155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28272024-02-20T22:42:56.720Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.227957874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28282024-02-20T22:42:56.720Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.228025961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28292024-02-20T22:42:56.720Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.228108138Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28302024-02-20T22:42:56.720Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.228146317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28312024-02-20T22:42:56.720Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.228187705Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28322024-02-20T22:42:56.720Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.22889014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28332024-02-20T22:42:56.720Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.228937539Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28342024-02-20T22:42:56.720Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.228990267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28352024-02-20T22:42:56.720Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.229066484Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28362024-02-20T22:42:56.720Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.229121942Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28372024-02-20T22:42:56.720Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.229154201Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28382024-02-20T22:42:56.720Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.229562026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28392024-02-20T22:42:56.720Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.229602185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28402024-02-20T22:42:56.720Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.229712921Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28412024-02-20T22:42:56.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230214253Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28422024-02-20T22:42:56.720Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230270831Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28432024-02-20T22:42:56.720Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.2303103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28442024-02-20T22:42:56.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230335279Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28452024-02-20T22:42:56.721Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230363607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28462024-02-20T22:42:56.721Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230387557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28472024-02-20T22:42:56.721Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230414016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28482024-02-20T22:42:56.721Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230438565Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28492024-02-20T22:42:56.721Z{"msg":"[0]R flush_numbers[0..12]: [49, 49, 47, 49, 49, 48, 48, 45, 48, 49, 49, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230466224Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28502024-02-20T22:42:56.721Z{"msg":"[0]R generation[0..12]: [25, 25, 24, 25, 25, 24, 24, 23, 24, 25, 25, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230491693Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28512024-02-20T22:42:56.721ZWait for a query_work_queue command to finish before sending IO
28522024-02-20T22:42:56.721Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230526812Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28532024-02-20T22:42:56.721Z{"msg":"[1]R flush_numbers[0..12]: [49, 49, 47, 49, 49, 48, 48, 45, 48, 49, 49, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230559471Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28542024-02-20T22:42:56.721Z{"msg":"[1]R generation[0..12]: [25, 25, 24, 25, 25, 24, 24, 23, 24, 25, 25, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.23057852Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28552024-02-20T22:42:56.721Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230614559Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28562024-02-20T22:42:56.721Z{"msg":"[2]R flush_numbers[0..12]: [49, 49, 47, 49, 49, 48, 48, 45, 48, 49, 49, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230631178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28572024-02-20T22:42:56.721Z{"msg":"[2]R generation[0..12]: [25, 25, 24, 25, 25, 24, 24, 23, 24, 25, 25, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230665467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28582024-02-20T22:42:56.721Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230690476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28592024-02-20T22:42:56.721Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230714415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28602024-02-20T22:42:56.721Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230738104Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28612024-02-20T22:42:56.721Z{"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230760513Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28622024-02-20T22:42:56.721Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230783383Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28632024-02-20T22:42:56.721Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230812822Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28642024-02-20T22:42:56.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230835161Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28652024-02-20T22:42:56.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.23086297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28662024-02-20T22:42:56.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230886299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28672024-02-20T22:42:56.721Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230912748Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28682024-02-20T22:42:56.721Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230934637Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28692024-02-20T22:42:56.721Z{"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28702024-02-20T22:42:56.722Zmsg":"7db37de8-c097-4717-93bd-113eff538192 is now active with session: 13e2b267-c705-4c4e-95e0-c0c01f2bf2c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230965506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28712024-02-20T22:42:56.722Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.230988715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239,"session_id":"13e2b267-c705-4c4e-95e0-c0c01f2bf2c9"}
28722024-02-20T22:42:56.722Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:42:56.231010575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239}
28732024-02-20T22:42:56.722ZLoading write count information from file "/tmp/test_repair_verify.data"
28742024-02-20T22:42:56.722ZRun Repair workload
28752024-02-20T22:42:56.722Z01/30 Write block 268 len 8192 data: 4 4
28762024-02-20T22:42:56.722Z02/30 Write block 355 len 36864 data: 2 2 2 3 4 5 5 5 7
28772024-02-20T22:42:56.734Z03/30 Write block 315 len 36864 data: 4 3 3 2 2 3 3 4 2
28782024-02-20T22:42:56.734Z04/30 Read block 169 len 4096
28792024-02-20T22:42:56.734Z05/30 Write block 59 len 20480 data: 4 2 2 2 2
28802024-02-20T22:42:56.734Z06/30 Read block 434 len 12288
28812024-02-20T22:42:56.734Z07/30 Write block 205 len 40960 data: 4 4 4 4 4 3 3 3 2 2
28822024-02-20T22:42:56.734Z08/30 Write block 286 len 4096 data: 2
28832024-02-20T22:42:56.734Z09/30 Write block 448 len 32768 data: 2 2 2 2 3 3 3 3
28842024-02-20T22:42:56.734Z10/30 Write block 70 len 36864 data: 4 3 3 3 4 3 3 3 3
28852024-02-20T22:42:56.734Z11/30 Write block 365 len 4096 data: 9
28862024-02-20T22:42:56.735Z12/30 Read block 201 len 16384
28872024-02-20T22:42:56.735Z13/30 Flush
28882024-02-20T22:42:56.735Z14/30 Write block 232 len 40960 data: 4 4 4 4 3 3 3 3 3 2
28892024-02-20T22:42:56.735Z15/30 Write block 301 len 12288 data: 3 3 3
28902024-02-20T22:42:56.735Z16/30 Write block 122 len 4096 data: 3
28912024-02-20T22:42:56.735Z17/30 Read block 289 len 36864
28922024-02-20T22:42:56.735Z18/30 Read block 395 len 4096
28932024-02-20T22:42:56.735Z19/30 Read block 18 len 4096
28942024-02-20T22:42:56.735Z20/30 Write block 25 len 8192 data: 5 5
28952024-02-20T22:42:56.735Z21/30 Read block 63 len 28672
28962024-02-20T22:42:56.735Z22/30 Write block 42 len 32768 data: 3 2 2 2 2 3 3 2
28972024-02-20T22:42:56.735Z23/30 Read block 296 len 12288
28982024-02-20T22:42:56.735Z24/30 Write block 149 len 16384 data: 4 6 6 5
28992024-02-20T22:42:56.735Z25/30 Read block 396 len 36864
29002024-02-20T22:42:56.735Z26/30 Write block 475 len 24576 data: 2 2 2 2 3 4
29012024-02-20T22:42:56.735Z27/30 Read block 227 len 28672
29022024-02-20T22:42:56.735Z28/30 Write block 51 len 24576 data: 2 3 2 2 2 2
29032024-02-20T22:42:56.735Z29/30 Read block 12 len 36864
29042024-02-20T22:42:56.735Z30/30 Write block 281 len 28672 data: 3 3 3 3 2 3 2
29052024-02-20T22:42:56.735Z----------------------------------------------------------------
29062024-02-20T22:42:56.735Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29072024-02-20T22:42:56.735ZStates: Active Active Active
29082024-02-20T22:42:56.735ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29092024-02-20T22:42:56.735Z 1 Acked 1000 Write 2 Done Done Done false
29102024-02-20T22:42:56.735Z 2 Acked 1001 Write 9 Done Done Done false
29112024-02-20T22:42:56.735Z 3 Acked 1002 Write 9 Sent Done Done false
29122024-02-20T22:42:56.736Z 4 Acked 1003 Read 1 Sent Done Done false
29132024-02-20T22:42:56.736Z 5 Acked 1004 Write 5 Sent Done Done false
29142024-02-20T22:42:56.736Z 6 Acked 1005 Read 3 Sent Done Done false
29152024-02-20T22:42:56.736Z 7 Acked 1006 Write 10 Sent Done Done false
29162024-02-20T22:42:56.736Z 8 Acked 1007 Write 1 Sent Done Done false
29172024-02-20T22:42:56.736Z 9 Acked 1008 Write 8 Sent Done Done false
29182024-02-20T22:42:56.736Z 10 Acked 1009 Write 9 Sent Done Done false
29192024-02-20T22:42:56.736Z 11 Acked 1010 Write 1 Sent Done Done false
29202024-02-20T22:42:56.736Z 12 Acked 1011 Read 4 Sent Done Done false
29212024-02-20T22:42:56.736Z 13 Acked 1012 Flush 0 Sent Done Done false
29222024-02-20T22:42:56.736Z 14 Acked 1013 Write 10 Sent Done Done false
29232024-02-20T22:42:56.736Z 15 Acked 1014 Write 3 Sent Done Done false
29242024-02-20T22:42:56.736Z 16 Acked 1015 Write 1 Sent Done Done false
29252024-02-20T22:42:56.736Z 17 Acked 1016 Read 9 Sent Done Done false
29262024-02-20T22:42:56.736Z 18 Acked 1017 Read 1 Sent Done Done false
29272024-02-20T22:42:56.736Z 19 Acked 1018 Read 1 Sent Done Done false
29282024-02-20T22:42:56.736Z 20 Acked 1019 Write 2 Sent Done Done false
29292024-02-20T22:42:56.736Z 21 Acked 1020 Read 7 Sent Done Done false
29302024-02-20T22:42:56.736Z 22 Acked 1021 Write 8 Sent Done Done false
29312024-02-20T22:42:56.736Z 23 Acked 1022 Read 3 Sent Done Done false
29322024-02-20T22:42:56.736Z 24 Acked 1023 Write 4 Sent Done Done false
29332024-02-20T22:42:56.736Z 25 Acked 1024 Read 9 Sent Done Done false
29342024-02-20T22:42:56.736Z 26 Acked 1025 Write 6 Sent Done Done false
29352024-02-20T22:42:56.736Z 27 Acked 1026 Read 7 Sent Done Done false
29362024-02-20T22:42:56.737Z 28 Acked 1027 Write 6 Sent Done Done false
29372024-02-20T22:42:56.737Z 29 Acked 1028 Read 9 Sent Done Done false
29382024-02-20T22:42:56.737Z 30 Acked 1029 Write 7 Sent Sent Sent false
29392024-02-20T22:42:56.737Z STATES DS:0 DS:1 DS:2 TOTAL
29402024-02-20T22:42:56.737Z New 0 0 0 0
29412024-02-20T22:42:56.737Z Sent 28 1 1 30
29422024-02-20T22:42:56.737Z Done 2 29 29 60
29432024-02-20T22:42:56.737Z Skipped 0 0 0 0
29442024-02-20T22:42:56.737Z Error 0 0 0 0
29452024-02-20T22:42:56.737ZLast Flush: 0 1012 1012
29462024-02-20T22:42:56.737ZDownstairs last five completed:
29472024-02-20T22:42:56.737ZUpstairs last five completed: 30 29 28 27 26
29482024-02-20T22:42:56.737Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:42:56.260367404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1239}
29492024-02-20T22:42:56.737ZWrote out file "/tmp/test_repair_verify.data"
29502024-02-20T22:42:56.737Z
29512024-02-20T22:42:56.743ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29522024-02-20T22:42:56.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29532024-02-20T22:42:56.750Z 0 000-019 25 25 25 49 49 49 F F F
29542024-02-20T22:42:56.750Z 1 020-039 25 25 25 49 49 49 F T T <---
29552024-02-20T22:42:56.750Z 2 040-059 24 26 26 47 50 50 F T T <---
29562024-02-20T22:42:56.750Z 3 060-079 25 26 26 49 50 50 F F F <---
29572024-02-20T22:42:56.750Z 4 080-099 25 25 25 49 49 49 F F F
29582024-02-20T22:42:56.750Z 5 100-119 24 24 24 48 48 48 F F F
29592024-02-20T22:42:56.750Z 6 120-139 24 24 24 48 48 48 F T T <---
29602024-02-20T22:42:56.750Z 7 140-159 23 23 23 45 45 45 F T T <---
29612024-02-20T22:42:56.750Z 8 160-179 24 24 24 48 48 48 F F F
29622024-02-20T22:42:56.750Z 9 180-199 25 25 25 49 49 49 F F F
29632024-02-20T22:42:56.751Z 10 200-219 25 26 26 49 50 50 F F F <---
29642024-02-20T22:42:56.751Z 11 220-239 14 14 14 29 29 29 F T T <---
29652024-02-20T22:42:56.751Z 12 240-259 16 16 16 33 33 33 F T T <---
29662024-02-20T22:42:56.751Z 13 260-279 25 26 26 49 50 50 T F F <---
29672024-02-20T22:42:56.751Z 14 280-299 20 26 26 39 50 50 F T T <---
29682024-02-20T22:42:56.751Z 15 300-319 23 26 26 45 50 50 F T T <---
29692024-02-20T22:42:56.751Z 16 320-339 25 26 26 49 50 50 F F F <---
29702024-02-20T22:42:56.751Z 17 340-359 25 26 26 49 50 50 T F F <---
29712024-02-20T22:42:56.751Z 18 360-379 25 26 26 49 50 50 T F F <---
29722024-02-20T22:42:56.751Z 19 380-399 23 23 23 45 45 45 F F F
29732024-02-20T22:42:56.751Z 20 400-419 25 25 25 49 49 49 F F F
29742024-02-20T22:42:56.751Z 21 420-439 19 19 19 38 38 38 F F F
29752024-02-20T22:42:56.751Z 22 440-459 14 26 26 27 50 50 F F F <---
29762024-02-20T22:42:56.751Z 23 460-479 22 22 22 44 44 44 F T T <---
29772024-02-20T22:42:56.751Z 24 480-499 22 22 22 44 44 44 F T T <---
29782024-02-20T22:42:56.751Z 25 500-519 21 21 21 43 43 43 F F F
29792024-02-20T22:42:56.751Z 26 520-539 19 19 19 38 38 38 F F F
29802024-02-20T22:42:56.751Z 27 540-559 23 23 23 45 45 45 F F F
29812024-02-20T22:42:56.751Z 28 560-579 12 12 12 23 23 23 F F F
29822024-02-20T22:42:56.751Z 29 580-599 25 25 25 49 49 49 F F F
29832024-02-20T22:42:56.751ZMax gen: 26, Max flush: 50
29842024-02-20T22:42:56.758ZError: Difference in extent metadata found!
29852024-02-20T22:42:56.758ZA Difference in extent metadata is expected here
29862024-02-20T22:42:56.758ZOn loop 13
29872024-02-20T22:42:56.758Z
29882024-02-20T22:42:56.758ZVerifying data now
29892024-02-20T22:43:03.618ZNo change after verify
29902024-02-20T22:43:03.618ZLoop: 13 Downstairs dump after verify (and repair):
29912024-02-20T22:43:03.638ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29922024-02-20T22:43:03.638Z 0 000-019 25 25 25 49 49 49 F F F
29932024-02-20T22:43:03.638Z 1 020-039 27 27 27 51 51 51 F F F
29942024-02-20T22:43:03.638Z 2 040-059 27 27 27 51 51 51 F F F
29952024-02-20T22:43:03.638Z 3 060-079 26 26 26 50 50 50 F F F
29962024-02-20T22:43:03.638Z 4 080-099 25 25 25 49 49 49 F F F
29972024-02-20T22:43:03.645Z 5 100-119 24 24 24 48 48 48 F F F
29982024-02-20T22:43:03.645Z 6 120-139 27 27 27 51 51 51 F F F
29992024-02-20T22:43:03.645Z 7 140-159 27 27 27 51 51 51 F F F
30002024-02-20T22:43:03.645Z 8 160-179 24 24 24 48 48 48 F F F
30012024-02-20T22:43:03.645Z 9 180-199 25 25 25 49 49 49 F F F
30022024-02-20T22:43:03.645Z 10 200-219 26 26 26 50 50 50 F F F
30032024-02-20T22:43:03.646Z 11 220-239 27 27 27 51 51 51 F F F
30042024-02-20T22:43:03.646Z 12 240-259 27 27 27 51 51 51 F F F
30052024-02-20T22:43:03.646Z 13 260-279 26 26 26 50 50 50 F F F
30062024-02-20T22:43:03.646Z 14 280-299 27 27 27 51 51 51 F F F
30072024-02-20T22:43:03.646Z 15 300-319 27 27 27 51 51 51 F F F
30082024-02-20T22:43:03.646Z 16 320-339 26 26 26 50 50 50 F F F
30092024-02-20T22:43:03.646Z 17 340-359 26 26 26 50 50 50 F F F
30102024-02-20T22:43:03.646Z 18 360-379 26 26 26 50 50 50 F F F
30112024-02-20T22:43:03.646Z 19 380-399 23 23 23 45 45 45 F F F
30122024-02-20T22:43:03.646Z 20 400-419 25 25 25 49 49 49 F F F
30132024-02-20T22:43:03.646Z 21 420-439 19 19 19 38 38 38 F F F
30142024-02-20T22:43:03.646Z 22 440-459 26 26 26 50 50 50 F F F
30152024-02-20T22:43:03.646Z 23 460-479 27 27 27 51 51 51 F F F
30162024-02-20T22:43:03.647Z 24 480-499 27 27 27 51 51 51 F F F
30172024-02-20T22:43:03.647Z 25 500-519 21 21 21 43 43 43 F F F
30182024-02-20T22:43:03.647Z 26 520-539 19 19 19 38 38 38 F F F
30192024-02-20T22:43:03.647Z 27 540-559 23 23 23 45 45 45 F F F
30202024-02-20T22:43:03.647Z 28 560-579 12 12 12 23 23 23 F F F
30212024-02-20T22:43:03.647Z 29 580-599 25 25 25 49 49 49 F F F
30222024-02-20T22:43:03.647ZMax gen: 27, Max flush: 51
30232024-02-20T22:43:03.647Z
30242024-02-20T22:43:03.647ZBegin loop 14
30252024-02-20T22:43:03.647ZDownstairs to restart: 2
30262024-02-20T22:43:03.665Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187167712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247}
30272024-02-20T22:43:03.665Z{"Crucible runtime is spawned
30282024-02-20T22:43:03.665Zmsg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187562678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247}
30292024-02-20T22:43:03.665Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187607096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247}
30302024-02-20T22:43:03.665Z{"msg":"Crucible 23313be5-2b0a-4f58-a5eb-de1a09270a82 has session id: 9b967683-2afa-4a0c-b88f-555fb56ba423","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187624866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30312024-02-20T22:43:03.665Z{"msg":"Upstairs opts: Upstairs UUID: 23313be5-2b0a-4f58-a5eb-de1a09270a82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187656425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30322024-02-20T22:43:03.665Z{"msg":"Crucible stats registered with UUID: 23313be5-2b0a-4f58-a5eb-de1a09270a82","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187671584Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30332024-02-20T22:43:03.665Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187685464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247}
30342024-02-20T22:43:03.665Z{"msg":"23313be5-2b0a-4f58-a5eb-de1a09270a82 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187705463Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30352024-02-20T22:43:03.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187718532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30362024-02-20T22:43:03.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187732792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30372024-02-20T22:43:03.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187746951Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30382024-02-20T22:43:03.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187761421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30392024-02-20T22:43:03.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187774841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30402024-02-20T22:43:03.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.18778953Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30412024-02-20T22:43:03.666Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187860638Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30422024-02-20T22:43:03.666Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187917535Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30432024-02-20T22:43:03.666Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.187968183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30442024-02-20T22:43:03.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188135208Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30452024-02-20T22:43:03.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188210065Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30462024-02-20T22:43:03.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188255363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30472024-02-20T22:43:03.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188837363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30482024-02-20T22:43:03.666Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188884941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30492024-02-20T22:43:03.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.18891228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30502024-02-20T22:43:03.666Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188954218Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30512024-02-20T22:43:03.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.188996987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30522024-02-20T22:43:03.666Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.189036785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30532024-02-20T22:43:03.666Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.18945602Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30542024-02-20T22:43:03.666Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.189494839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30552024-02-20T22:43:03.666Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.189690762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30562024-02-20T22:43:03.666Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.189922364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30572024-02-20T22:43:03.666Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.189988791Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30582024-02-20T22:43:03.666Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.19002784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30592024-02-20T22:43:03.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190069578Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30602024-02-20T22:43:03.667Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190114297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30612024-02-20T22:43:03.667Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190142166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30622024-02-20T22:43:03.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190227723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30632024-02-20T22:43:03.667Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190276611Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30642024-02-20T22:43:03.667Z{"msg":"[0]R flush_numbers[0..12]: [49, 51, 51, 50, 49, 48, 51, 51, 48, 49, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190322139Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30652024-02-20T22:43:03.667Z{"msg":"[0]R generation[0..12]: [25, 27, 27, 26, 25, 24, 27, 27, 24, 25, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190340869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30662024-02-20T22:43:03.667Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190363088Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30672024-02-20T22:43:03.667Z{"msg":"[1]R flush_numbers[0..12]: [49, 51, 51, 50, 49, 48, 51, 51, 48, 49, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190377997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30682024-02-20T22:43:03.667Z{"msg":"[1]R generation[0..12]: [25, 27, 27, 26, 25, 24, 27, 27, 24, 25, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190392467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30692024-02-20T22:43:03.667ZWait for a query_work_queue command to finish before sending IO
30702024-02-20T22:43:03.667Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190414056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30712024-02-20T22:43:03.667Z{"msg":"[2]R flush_numbers[0..12]: [49, 51, 51, 50, 49, 48, 51, 51, 48, 49, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190429186Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30722024-02-20T22:43:03.667Z{"msg":"[2]R generation[0..12]: [25, 27, 27, 26, 25, 24, 27, 27, 24, 25, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190442995Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30732024-02-20T22:43:03.667Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190464844Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30742024-02-20T22:43:03.667Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190478584Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30752024-02-20T22:43:03.667Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190492903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30762024-02-20T22:43:03.667Z{"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190513112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30772024-02-20T22:43:03.667Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190526902Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30782024-02-20T22:43:03.667Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190540211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30792024-02-20T22:43:03.667Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190559441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30802024-02-20T22:43:03.667Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.19057312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30812024-02-20T22:43:03.667Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.19058718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30822024-02-20T22:43:03.667Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190601529Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30832024-02-20T22:43:03.667Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190614219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30842024-02-20T22:43:03.667Z{"msg":"23313be5-2b0a-4f58-a5eb-de1a09270a82 is now active with session: 9b967683-2afa-4a0c-b88f-555fb56ba423","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190634868Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30852024-02-20T22:43:03.668Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190647458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247,"session_id":"9b967683-2afa-4a0c-b88f-555fb56ba423"}
30862024-02-20T22:43:03.668Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:03.190659777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247}
30872024-02-20T22:43:03.668ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30882024-02-20T22:43:03.668ZLoading write count information from file "/tmp/test_repair_verify.data"
30892024-02-20T22:43:03.668ZRun Repair workload
30902024-02-20T22:43:03.668Z01/30 Write block 470 len 4096 data: 4
30912024-02-20T22:43:03.672Z02/30 Read block 317 len 24576
30922024-02-20T22:43:03.672Z03/30 Write block 347 len 16384 data: 3 3 3 3
30932024-02-20T22:43:03.672Z04/30 Read block 103 len 16384
30942024-02-20T22:43:03.673Z05/30 Flush
30952024-02-20T22:43:03.673Z06/30 Write block 27 len 16384 data: 5 6 5 6
30962024-02-20T22:43:03.673Z07/30 Read block 264 len 20480
30972024-02-20T22:43:03.673Z08/30 Write block 135 len 4096 data: 4
30982024-02-20T22:43:03.673Z09/30 Write block 510 len 40960 data: 3 3 2 2 2 2 2 2 2 2
30992024-02-20T22:43:03.673Z10/30 Write block 460 len 32768 data: 3 3 6 6 5 6 6 6
31002024-02-20T22:43:03.673Z11/30 Read block 273 len 36864
31012024-02-20T22:43:03.673Z12/30 Write block 365 len 8192 data: 10 8
31022024-02-20T22:43:03.673Z13/30 Flush
31032024-02-20T22:43:03.691Z14/30 Flush
31042024-02-20T22:43:03.696Z15/30 Write block 460 len 40960 data: 4 4 7 7 6 7 7 7 6 6
31052024-02-20T22:43:03.696Z16/30 Read block 313 len 24576
31062024-02-20T22:43:03.696Z17/30 Flush
31072024-02-20T22:43:03.701Z18/30 Write block 334 len 16384 data: 4 4 4 4
31082024-02-20T22:43:03.701Z19/30 Write block 448 len 12288 data: 3 3 3
31092024-02-20T22:43:03.701Z20/30 Write block 492 len 40960 data: 2 3 3 3 4 4 3 3 3 3
31102024-02-20T22:43:03.701Z21/30 Flush
31112024-02-20T22:43:03.760Z22/30 Write block 443 len 40960 data: 2 2 2 2 2 4 4 4 3 4
31122024-02-20T22:43:03.765Z23/30 Read block 209 len 40960
31132024-02-20T22:43:03.765Z24/30 Read block 93 len 24576
31142024-02-20T22:43:03.765Z25/30 Read block 20 len 24576
31152024-02-20T22:43:03.765Z26/30 Write block 331 len 8192 data: 4 5
31162024-02-20T22:43:03.765Z27/30 Write block 515 len 4096 data: 3
31172024-02-20T22:43:03.765Z28/30 Read block 513 len 12288
31182024-02-20T22:43:03.770Z29/30 Read block 225 len 16384
31192024-02-20T22:43:03.770Z30/30 Read block 363 len 4096
31202024-02-20T22:43:03.770Z----------------------------------------------------------------
31212024-02-20T22:43:03.770Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31222024-02-20T22:43:03.770ZStates: Active Active Active
31232024-02-20T22:43:03.770ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31242024-02-20T22:43:03.770Z 1 Acked 1000 Write 1 Done Done Done false
31252024-02-20T22:43:03.771Z 2 Acked 1001 Read 6 Done Done Done false
31262024-02-20T22:43:03.771Z 3 Acked 1002 Write 4 Done Done Sent false
31272024-02-20T22:43:03.771Z 4 Acked 1003 Read 4 Done Done Sent false
31282024-02-20T22:43:03.771Z 5 Acked 1004 Flush 0 Done Done Sent false
31292024-02-20T22:43:03.771Z 6 Acked 1005 Write 4 Done Done Sent false
31302024-02-20T22:43:03.771Z 7 Acked 1006 Read 5 Done Done Sent false
31312024-02-20T22:43:03.771Z 8 Acked 1007 Write 1 Done Done Sent false
31322024-02-20T22:43:03.771Z 9 Acked 1008 Write 10 Done Done Sent false
31332024-02-20T22:43:03.771Z 10 Acked 1009 Write 8 Done Done Sent false
31342024-02-20T22:43:03.771Z 11 Acked 1010 Read 9 Done Done Sent false
31352024-02-20T22:43:03.771Z 12 Acked 1011 Write 2 Done Done Sent false
31362024-02-20T22:43:03.771Z 13 Acked 1012 Flush 0 Done Done Sent false
31372024-02-20T22:43:03.771Z 14 Acked 1013 Flush 0 Done Done Sent false
31382024-02-20T22:43:03.771Z 15 Acked 1014 Write 10 Done Done Sent false
31392024-02-20T22:43:03.771Z 16 Acked 1015 Read 6 Done Done Sent false
31402024-02-20T22:43:03.771Z 17 Acked 1016 Flush 0 Done Done Sent false
31412024-02-20T22:43:03.771Z 18 Acked 1017 Write 4 Done Done Sent false
31422024-02-20T22:43:03.771Z 19 Acked 1018 Write 3 Done Done Sent false
31432024-02-20T22:43:03.771Z 20 Acked 1019 Write 10 Done Done Sent false
31442024-02-20T22:43:03.771Z 21 Acked 1020 Flush 0 Done Done Sent false
31452024-02-20T22:43:03.771Z 22 Acked 1021 Write 10 Done Done Sent false
31462024-02-20T22:43:03.771Z 23 Acked 1022 Read 10 Done Done Sent false
31472024-02-20T22:43:03.771Z 24 Acked 1023 Read 6 Done Done Sent false
31482024-02-20T22:43:03.771Z 25 Acked 1024 Read 6 Done Done Sent false
31492024-02-20T22:43:03.772Z 26 Acked 1025 Write 2 Done Done Sent false
31502024-02-20T22:43:03.772Z 27 Acked 1026 Write 1 Done Done Sent false
31512024-02-20T22:43:03.772Z 28 Acked 1027 Read 3 Done Done Sent false
31522024-02-20T22:43:03.772Z 29 Acked 1028 Read 4 Done Done Sent false
31532024-02-20T22:43:03.772Z 30 Acked 1029 Read 1 Done Sent Sent false
31542024-02-20T22:43:03.772Z STATES DS:0 DS:1 DS:2 TOTAL
31552024-02-20T22:43:03.772Z New 0 0 0 0
31562024-02-20T22:43:03.772Z Sent 0 1 28 29
31572024-02-20T22:43:03.772Z Done 30 29 2 61
31582024-02-20T22:43:03.772Z Skipped 0 0 0 0
31592024-02-20T22:43:03.772Z Error 0 0 0 0
31602024-02-20T22:43:03.772ZLast Flush: 1020 1020 0
31612024-02-20T22:43:03.772ZDownstairs last five completed:
31622024-02-20T22:43:03.772ZUpstairs last five completed: 30 29 28 27 26
31632024-02-20T22:43:03.772Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:03.304633479Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1247}
31642024-02-20T22:43:03.772ZWrote out file "/tmp/test_repair_verify.data"
31652024-02-20T22:43:03.778Z
31662024-02-20T22:43:03.778ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31672024-02-20T22:43:03.791ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31682024-02-20T22:43:03.791Z 0 000-019 25 25 25 49 49 49 F F F
31692024-02-20T22:43:03.791Z 1 020-039 28 28 27 53 53 51 F F F <---
31702024-02-20T22:43:03.791Z 2 040-059 27 27 27 51 51 51 F F F
31712024-02-20T22:43:03.791Z 3 060-079 26 26 26 50 50 50 F F F
31722024-02-20T22:43:03.791Z 4 080-099 25 25 25 49 49 49 F F F
31732024-02-20T22:43:03.791Z 5 100-119 24 24 24 48 48 48 F F F
31742024-02-20T22:43:03.791Z 6 120-139 28 28 27 53 53 51 F F F <---
31752024-02-20T22:43:03.791Z 7 140-159 27 27 27 51 51 51 F F F
31762024-02-20T22:43:03.796Z 8 160-179 24 24 24 48 48 48 F F F
31772024-02-20T22:43:03.796Z 9 180-199 25 25 25 49 49 49 F F F
31782024-02-20T22:43:03.796Z 10 200-219 26 26 26 50 50 50 F F F
31792024-02-20T22:43:03.796Z 11 220-239 27 27 27 51 51 51 F F F
31802024-02-20T22:43:03.796Z 12 240-259 27 27 27 51 51 51 F F F
31812024-02-20T22:43:03.796Z 13 260-279 26 26 26 50 50 50 F F F
31822024-02-20T22:43:03.796Z 14 280-299 27 27 27 51 51 51 F F F
31832024-02-20T22:43:03.796Z 15 300-319 27 27 27 51 51 51 F F F
31842024-02-20T22:43:03.796Z 16 320-339 28 28 26 56 56 50 T T F <---
31852024-02-20T22:43:03.796Z 17 340-359 28 28 26 52 52 50 F F F <---
31862024-02-20T22:43:03.796Z 18 360-379 28 28 26 53 53 50 F F F <---
31872024-02-20T22:43:03.796Z 19 380-399 23 23 23 45 45 45 F F F
31882024-02-20T22:43:03.796Z 20 400-419 25 25 25 49 49 49 F F F
31892024-02-20T22:43:03.796Z 21 420-439 19 19 19 38 38 38 F F F
31902024-02-20T22:43:03.796Z 22 440-459 28 28 26 56 56 50 T T F <---
31912024-02-20T22:43:03.796Z 23 460-479 28 28 27 55 55 51 F F T <---
31922024-02-20T22:43:03.796Z 24 480-499 28 28 27 56 56 51 F F F <---
31932024-02-20T22:43:03.796Z 25 500-519 28 28 21 56 56 43 T T F <---
31942024-02-20T22:43:03.796Z 26 520-539 19 19 19 38 38 38 F F F
31952024-02-20T22:43:03.796Z 27 540-559 23 23 23 45 45 45 F F F
31962024-02-20T22:43:03.796Z 28 560-579 12 12 12 23 23 23 F F F
31972024-02-20T22:43:03.796Z 29 580-599 25 25 25 49 49 49 F F F
31982024-02-20T22:43:03.797ZMax gen: 28, Max flush: 56
31992024-02-20T22:43:03.797ZError: Difference in extent metadata found!
32002024-02-20T22:43:03.797ZA Difference in extent metadata is expected here
32012024-02-20T22:43:03.797ZOn loop 14
32022024-02-20T22:43:03.797Z
32032024-02-20T22:43:03.797ZVerifying data now
32042024-02-20T22:43:09.733ZNo change after verify
32052024-02-20T22:43:09.733ZLoop: 14 Downstairs dump after verify (and repair):
32062024-02-20T22:43:09.748ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32072024-02-20T22:43:09.748Z 0 000-019 25 25 25 49 49 49 F F F
32082024-02-20T22:43:09.748Z 1 020-039 28 28 28 53 53 53 F F F
32092024-02-20T22:43:09.748Z 2 040-059 27 27 27 51 51 51 F F F
32102024-02-20T22:43:09.748Z 3 060-079 26 26 26 50 50 50 F F F
32112024-02-20T22:43:09.748Z 4 080-099 25 25 25 49 49 49 F F F
32122024-02-20T22:43:09.748Z 5 100-119 24 24 24 48 48 48 F F F
32132024-02-20T22:43:09.748Z 6 120-139 28 28 28 53 53 53 F F F
32142024-02-20T22:43:09.755Z 7 140-159 27 27 27 51 51 51 F F F
32152024-02-20T22:43:09.755Z 8 160-179 24 24 24 48 48 48 F F F
32162024-02-20T22:43:09.755Z 9 180-199 25 25 25 49 49 49 F F F
32172024-02-20T22:43:09.755Z 10 200-219 26 26 26 50 50 50 F F F
32182024-02-20T22:43:09.755Z 11 220-239 27 27 27 51 51 51 F F F
32192024-02-20T22:43:09.755Z 12 240-259 27 27 27 51 51 51 F F F
32202024-02-20T22:43:09.755Z 13 260-279 26 26 26 50 50 50 F F F
32212024-02-20T22:43:09.755Z 14 280-299 27 27 27 51 51 51 F F F
32222024-02-20T22:43:09.755Z 15 300-319 27 27 27 51 51 51 F F F
32232024-02-20T22:43:09.755Z 16 320-339 29 29 29 57 57 57 F F F
32242024-02-20T22:43:09.755Z 17 340-359 28 28 28 52 52 52 F F F
32252024-02-20T22:43:09.755Z 18 360-379 28 28 28 53 53 53 F F F
32262024-02-20T22:43:09.756Z 19 380-399 23 23 23 45 45 45 F F F
32272024-02-20T22:43:09.756Z 20 400-419 25 25 25 49 49 49 F F F
32282024-02-20T22:43:09.756Z 21 420-439 19 19 19 38 38 38 F F F
32292024-02-20T22:43:09.756Z 22 440-459 29 29 29 57 57 57 F F F
32302024-02-20T22:43:09.756Z 23 460-479 28 28 28 55 55 55 F F F
32312024-02-20T22:43:09.756Z 24 480-499 28 28 28 56 56 56 F F F
32322024-02-20T22:43:09.756Z 25 500-519 29 29 29 57 57 57 F F F
32332024-02-20T22:43:09.756Z 26 520-539 19 19 19 38 38 38 F F F
32342024-02-20T22:43:09.756Z 27 540-559 23 23 23 45 45 45 F F F
32352024-02-20T22:43:09.756Z 28 560-579 12 12 12 23 23 23 F F F
32362024-02-20T22:43:09.756Z 29 580-599 25 25 25 49 49 49 F F F
32372024-02-20T22:43:09.756ZMax gen: 29, Max flush: 57
32382024-02-20T22:43:09.757Z
32392024-02-20T22:43:09.757ZBegin loop 15
32402024-02-20T22:43:09.757ZDownstairs to restart: 1
32412024-02-20T22:43:09.769Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306294801Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255}
32422024-02-20T22:43:09.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306566952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255}
32432024-02-20T22:43:09.780Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
32442024-02-20T22:43:09.780Z,"time":"2024-02-20T22:43:09.306665758Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255}
32452024-02-20T22:43:09.780Z{"msg":"Crucible 7ed77f42-252c-440a-88bf-ec9715a9cb47 has session id: 78cdc0a3-4383-4d28-a5c2-604aa20e544b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306767914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32462024-02-20T22:43:09.781Z{"msg":"Upstairs opts: Upstairs UUID: 7ed77f42-252c-440a-88bf-ec9715a9cb47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306815693Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32472024-02-20T22:43:09.781Z{"msg":"Crucible stats registered with UUID: 7ed77f42-252c-440a-88bf-ec9715a9cb47","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306853231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32482024-02-20T22:43:09.781Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30688797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255}
32492024-02-20T22:43:09.781Z{"msg":"7ed77f42-252c-440a-88bf-ec9715a9cb47 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306921649Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32502024-02-20T22:43:09.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.306957468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32512024-02-20T22:43:09.781Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307002206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32522024-02-20T22:43:09.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307045464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32532024-02-20T22:43:09.781Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307086083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32542024-02-20T22:43:09.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307121762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32552024-02-20T22:43:09.781Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307147191Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32562024-02-20T22:43:09.781Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30717255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32572024-02-20T22:43:09.781Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307201799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32582024-02-20T22:43:09.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307230408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32592024-02-20T22:43:09.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307270087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32602024-02-20T22:43:09.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307293366Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32612024-02-20T22:43:09.782Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307317175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32622024-02-20T22:43:09.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.307974531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32632024-02-20T22:43:09.782Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30801512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32642024-02-20T22:43:09.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.308070408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32652024-02-20T22:43:09.782Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.308125076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32662024-02-20T22:43:09.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.308155675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32672024-02-20T22:43:09.782Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.308203903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32682024-02-20T22:43:09.782Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.308594979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32692024-02-20T22:43:09.782Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.308646797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32702024-02-20T22:43:09.782Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30884432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32712024-02-20T22:43:09.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309105421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32722024-02-20T22:43:09.782Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309148129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32732024-02-20T22:43:09.782Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309176278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32742024-02-20T22:43:09.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309204317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32752024-02-20T22:43:09.782Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309230166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32762024-02-20T22:43:09.782Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309262445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32772024-02-20T22:43:09.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309286764Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32782024-02-20T22:43:09.782Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309314253Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32792024-02-20T22:43:09.783Z{"msg":"[0]R flush_numbers[0..12]: [49, 53, 51, 50, 49, 48, 53, 51, 48, 49, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309341022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32802024-02-20T22:43:09.783Z{"msg":"[0]R generation[0..12]: [25, 28, 27, 26, 25, 24, 28, 27, 24, 25, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309376211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32812024-02-20T22:43:09.783Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30940468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32822024-02-20T22:43:09.783Z{"msg":"[1]R flush_numbers[0..12]: [49, 53, 51, 50, 49, 48, 53, 51, 48, 49, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309436849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32832024-02-20T22:43:09.783Z{"msg":"[1]R generation[0..12]: [25, 28, 27, 26, 25, 24, 28, 27, 24, 25, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309463018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32842024-02-20T22:43:09.783Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309503636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32852024-02-20T22:43:09.783Z{"msg":"[2]R flush_numbers[0..12]: [49, 53, 51, 50, 49, 48, 53, 51, 48, 49, 50, 51]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
32862024-02-20T22:43:09.783Z,"time":"2024-02-20T22:43:09.309535765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32872024-02-20T22:43:09.783Z{"msg":"[2]R generation[0..12]: [25, 28, 27, 26, 25, 24, 28, 27, 24, 25, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309566004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32882024-02-20T22:43:09.783Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309598413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32892024-02-20T22:43:09.783Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309613833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32902024-02-20T22:43:09.783Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309642562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32912024-02-20T22:43:09.783Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30968559Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32922024-02-20T22:43:09.783Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309723819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32932024-02-20T22:43:09.783Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309746778Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32942024-02-20T22:43:09.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309779757Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32952024-02-20T22:43:09.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309817715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32962024-02-20T22:43:09.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309855834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32972024-02-20T22:43:09.783Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309893443Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32982024-02-20T22:43:09.783Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.309929661Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
32992024-02-20T22:43:09.784Z{"msg":"7ed77f42-252c-440a-88bf-ec9715a9cb47 is now active with session: 78cdc0a3-4383-4d28-a5c2-604aa20e544b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.30996658Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
33002024-02-20T22:43:09.784Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.310003629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255,"session_id":"78cdc0a3-4383-4d28-a5c2-604aa20e544b"}
33012024-02-20T22:43:09.784Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:09.310039247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33022024-02-20T22:43:09.784Z}
33032024-02-20T22:43:09.784ZLoading write count information from file "/tmp/test_repair_verify.data"
33042024-02-20T22:43:09.784ZRun Repair workload
33052024-02-20T22:43:09.784Z01/30 Write block 359 len 12288 data: 5 6 6
33062024-02-20T22:43:09.789Z02/30 Read block 33 len 24576
33072024-02-20T22:43:09.789Z03/30 Read block 74 len 40960
33082024-02-20T22:43:09.789Z04/30 Write block 84 len 32768 data: 3 2 2 2 2 2 3 3
33092024-02-20T22:43:09.789Z05/30 Write block 452 len 32768 data: 5 4 4 4 4 4 4 3
33102024-02-20T22:43:09.789Z06/30 Read block 281 len 20480
33112024-02-20T22:43:09.789Z07/30 Read block 496 len 28672
33122024-02-20T22:43:09.789Z08/30 Read block 469 len 20480
33132024-02-20T22:43:09.789Z09/30 Read block 269 len 24576
33142024-02-20T22:43:09.789Z10/30 Read block 394 len 32768
33152024-02-20T22:43:09.789Z11/30 Write block 139 len 8192 data: 4 4
33162024-02-20T22:43:09.789Z12/30 Read block 547 len 32768
33172024-02-20T22:43:09.789Z13/30 Write block 29 len 12288 data: 6 7 6
33182024-02-20T22:43:09.789Z14/30 Flush
33192024-02-20T22:43:09.789Z15/30 Write block 2 len 40960 data: 3 3 4 4 3 2 2 2 3 2
33202024-02-20T22:43:09.789Z16/30 Write block 347 len 40960 data: 4 4 4 4 3 2 2 2 3 3
33212024-02-20T22:43:09.794Z17/30 Flush
33222024-02-20T22:43:09.794Z18/30 Write block 49 len 24576 data: 3 2 3 4 3 3
33232024-02-20T22:43:09.794Z19/30 Write block 184 len 24576 data: 5 4 3 3 3 3
33242024-02-20T22:43:09.794Z20/30 Read block 575 len 20480
33252024-02-20T22:43:09.794Z21/30 Read block 127 len 20480
33262024-02-20T22:43:09.799Z22/30 Flush
33272024-02-20T22:43:09.799Z23/30 Write block 497 len 12288 data: 5 4 4
33282024-02-20T22:43:09.799Z24/30 Write block 143 len 4096 data: 4
33292024-02-20T22:43:09.800Z25/30 Write block 500 len 4096 data: 4
33302024-02-20T22:43:09.800Z26/30 Read block 56 len 12288
33312024-02-20T22:43:09.805Z27/30 Write block 409 len 16384 data: 7 7 7 7
33322024-02-20T22:43:09.805Z28/30 Read block 33 len 16384
33332024-02-20T22:43:09.805Z29/30 Write block 480 len 20480 data: 5 4 4 5 5
33342024-02-20T22:43:09.805Z30/30 Read block 538 len 36864
33352024-02-20T22:43:09.805Z----------------------------------------------------------------
33362024-02-20T22:43:09.805Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33372024-02-20T22:43:09.805ZStates: Active Active Active
33382024-02-20T22:43:09.805ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33392024-02-20T22:43:09.805Z 1 Acked 1000 Write 3 Done Sent Done false
33402024-02-20T22:43:09.805Z 2 Acked 1001 Read 6 Done Sent Done false
33412024-02-20T22:43:09.806Z 3 Acked 1002 Read 10 Done Sent Done false
33422024-02-20T22:43:09.806Z 4 Acked 1003 Write 8 Done Sent Done false
33432024-02-20T22:43:09.806Z 5 Acked 1004 Write 8 Done Sent Done false
33442024-02-20T22:43:09.806Z 6 Acked 1005 Read 5 Done Sent Done false
33452024-02-20T22:43:09.806Z 7 Acked 1006 Read 7 Done Sent Done false
33462024-02-20T22:43:09.806Z 8 Acked 1007 Read 5 Done Sent Done false
33472024-02-20T22:43:09.806Z 9 Acked 1008 Read 6 Done Sent Done false
33482024-02-20T22:43:09.806Z 10 Acked 1009 Read 8 Done Sent Done false
33492024-02-20T22:43:09.806Z 11 Acked 1010 Write 2 Done Sent Done false
33502024-02-20T22:43:09.806Z 12 Acked 1011 Read 8 Done Sent Done false
33512024-02-20T22:43:09.806Z 13 Acked 1012 Write 3 Done Sent Done false
33522024-02-20T22:43:09.806Z 14 Acked 1013 Flush 0 Done Sent Done false
33532024-02-20T22:43:09.806Z 15 Acked 1014 Write 10 Done Sent Done false
33542024-02-20T22:43:09.807Z 16 Acked 1015 Write 10 Done Sent Done false
33552024-02-20T22:43:09.807Z 17 Acked 1016 Flush 0 Done Sent Done false
33562024-02-20T22:43:09.807Z 18 Acked 1017 Write 6 Done Sent Done false
33572024-02-20T22:43:09.807Z 19 Acked 1018 Write 6 Done Sent Done false
33582024-02-20T22:43:09.807Z 20 Acked 1019 Read 5 Done Sent Done false
33592024-02-20T22:43:09.807Z 21 Acked 1020 Read 5 Done Sent Done false
33602024-02-20T22:43:09.807Z 22 Acked 1021 Flush 0 Done Sent Done false
33612024-02-20T22:43:09.807Z 23 Acked 1022 Write 3 Done Sent Done false
33622024-02-20T22:43:09.807Z 24 Acked 1023 Write 1 Done Sent Done false
33632024-02-20T22:43:09.807Z 25 Acked 1024 Write 1 Done Sent Done false
33642024-02-20T22:43:09.807Z 26 Acked 1025 Read 3 Done Sent Done false
33652024-02-20T22:43:09.807Z 27 Acked 1026 Write 4 Done Sent Done false
33662024-02-20T22:43:09.807Z 28 Acked 1027 Read 4 Done Sent Done false
33672024-02-20T22:43:09.808Z 29 Acked 1028 Write 5 Done Sent Done false
33682024-02-20T22:43:09.808Z 30 Acked 1029 Read 9 Done Sent Sent false
33692024-02-20T22:43:09.808Z STATES DS:0 DS:1 DS:2 TOTAL
33702024-02-20T22:43:09.808Z New 0 0 0 0
33712024-02-20T22:43:09.808Z Sent 0 30 1 31
33722024-02-20T22:43:09.808Z Done 30 0 29 59
33732024-02-20T22:43:09.808Z Skipped 0 0 0 0
33742024-02-20T22:43:09.808Z Error 0 0 0 0
33752024-02-20T22:43:09.808ZLast Flush: 1021 0 1021
33762024-02-20T22:43:09.808ZDownstairs last five completed:
33772024-02-20T22:43:09.808ZUpstairs last five completed: 30 29 28 27 26
33782024-02-20T22:43:09.808Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:09.339961807Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1255}
33792024-02-20T22:43:09.808ZWrote out file "/tmp/test_repair_verify.data"
33802024-02-20T22:43:09.813Z
33812024-02-20T22:43:09.813ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33822024-02-20T22:43:09.825ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33832024-02-20T22:43:09.826Z 0 000-019 30 25 30 59 49 59 F F F <---
33842024-02-20T22:43:09.826Z 1 020-039 30 28 30 58 53 58 F F F <---
33852024-02-20T22:43:09.826Z 2 040-059 30 27 30 60 51 60 F F F <---
33862024-02-20T22:43:09.826Z 3 060-079 26 26 26 50 50 50 F F F
33872024-02-20T22:43:09.826Z 4 080-099 30 25 30 58 49 58 F F F <---
33882024-02-20T22:43:09.826Z 5 100-119 24 24 24 48 48 48 F F F
33892024-02-20T22:43:09.826Z 6 120-139 30 28 30 58 53 58 F F F <---
33902024-02-20T22:43:09.826Z 7 140-159 30 27 30 58 51 58 T F T <---
33912024-02-20T22:43:09.833Z 8 160-179 24 24 24 48 48 48 F F F
33922024-02-20T22:43:09.833Z 9 180-199 30 25 30 60 49 60 F F F <---
33932024-02-20T22:43:09.833Z 10 200-219 26 26 26 50 50 50 F F F
33942024-02-20T22:43:09.833Z 11 220-239 27 27 27 51 51 51 F F F
33952024-02-20T22:43:09.833Z 12 240-259 27 27 27 51 51 51 F F F
33962024-02-20T22:43:09.834Z 13 260-279 26 26 26 50 50 50 F F F
33972024-02-20T22:43:09.834Z 14 280-299 27 27 27 51 51 51 F F F
33982024-02-20T22:43:09.834Z 15 300-319 27 27 27 51 51 51 F F F
33992024-02-20T22:43:09.834Z 16 320-339 29 29 29 57 57 57 F F F
34002024-02-20T22:43:09.834Z 17 340-359 30 28 30 59 52 59 F F F <---
34012024-02-20T22:43:09.834Z 18 360-379 30 28 30 58 53 58 F F F <---
34022024-02-20T22:43:09.834Z 19 380-399 23 23 23 45 45 45 F F F
34032024-02-20T22:43:09.834Z 20 400-419 25 25 25 49 49 49 T F T <---
34042024-02-20T22:43:09.834Z 21 420-439 19 19 19 38 38 38 F F F
34052024-02-20T22:43:09.834Z 22 440-459 30 29 30 58 57 58 F F F <---
34062024-02-20T22:43:09.834Z 23 460-479 28 28 28 55 55 55 F F F
34072024-02-20T22:43:09.834Z 24 480-499 28 28 28 56 56 56 T F T <---
34082024-02-20T22:43:09.835Z 25 500-519 29 29 29 57 57 57 T F T <---
34092024-02-20T22:43:09.835Z 26 520-539 19 19 19 38 38 38 F F F
34102024-02-20T22:43:09.835Z 27 540-559 23 23 23 45 45 45 F F F
34112024-02-20T22:43:09.835Z 28 560-579 12 12 12 23 23 23 F F F
34122024-02-20T22:43:09.835Z 29 580-599 25 25 25 49 49 49 F F F
34132024-02-20T22:43:09.835ZMax gen: 30, Max flush: 60
34142024-02-20T22:43:09.835ZError: Difference in extent metadata found!
34152024-02-20T22:43:09.835ZA Difference in extent metadata is expected here
34162024-02-20T22:43:09.835ZOn loop 15
34172024-02-20T22:43:09.835Z
34182024-02-20T22:43:09.840ZVerifying data now
34192024-02-20T22:43:15.243ZNo change after verify
34202024-02-20T22:43:15.243ZLoop: 15 Downstairs dump after verify (and repair):
34212024-02-20T22:43:15.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34222024-02-20T22:43:15.260Z 0 000-019 30 30 30 59 59 59 F F F
34232024-02-20T22:43:15.260Z 1 020-039 30 30 30 58 58 58 F F F
34242024-02-20T22:43:15.260Z 2 040-059 30 30 30 60 60 60 F F F
34252024-02-20T22:43:15.260Z 3 060-079 26 26 26 50 50 50 F F F
34262024-02-20T22:43:15.260Z 4 080-099 30 30 30 58 58 58 F F F
34272024-02-20T22:43:15.265Z 5 100-119 24 24 24 48 48 48 F F F
34282024-02-20T22:43:15.265Z 6 120-139 30 30 30 58 58 58 F F F
34292024-02-20T22:43:15.265Z 7 140-159 31 31 31 61 61 61 F F F
34302024-02-20T22:43:15.265Z 8 160-179 24 24 24 48 48 48 F F F
34312024-02-20T22:43:15.265Z 9 180-199 30 30 30 60 60 60 F F F
34322024-02-20T22:43:15.265Z 10 200-219 26 26 26 50 50 50 F F F
34332024-02-20T22:43:15.265Z 11 220-239 27 27 27 51 51 51 F F F
34342024-02-20T22:43:15.265Z 12 240-259 27 27 27 51 51 51 F F F
34352024-02-20T22:43:15.265Z 13 260-279 26 26 26 50 50 50 F F F
34362024-02-20T22:43:15.266Z 14 280-299 27 27 27 51 51 51 F F F
34372024-02-20T22:43:15.266Z 15 300-319 27 27 27 51 51 51 F F F
34382024-02-20T22:43:15.266Z 16 320-339 29 29 29 57 57 57 F F F
34392024-02-20T22:43:15.266Z 17 340-359 30 30 30 59 59 59 F F F
34402024-02-20T22:43:15.266Z 18 360-379 30 30 30 58 58 58 F F F
34412024-02-20T22:43:15.266Z 19 380-399 23 23 23 45 45 45 F F F
34422024-02-20T22:43:15.266Z 20 400-419 31 31 31 61 61 61 F F F
34432024-02-20T22:43:15.266Z 21 420-439 19 19 19 38 38 38 F F F
34442024-02-20T22:43:15.266Z 22 440-459 30 30 30 58 58 58 F F F
34452024-02-20T22:43:15.266Z 23 460-479 28 28 28 55 55 55 F F F
34462024-02-20T22:43:15.266Z 24 480-499 31 31 31 61 61 61 F F F
34472024-02-20T22:43:15.266Z 25 500-519 31 31 31 61 61 61 F F F
34482024-02-20T22:43:15.266Z 26 520-539 19 19 19 38 38 38 F F F
34492024-02-20T22:43:15.266Z 27 540-559 23 23 23 45 45 45 F F F
34502024-02-20T22:43:15.266Z 28 560-579 12 12 12 23 23 23 F F F
34512024-02-20T22:43:15.266Z 29 580-599 25 25 25 49 49 49 F F F
34522024-02-20T22:43:15.266ZMax gen: 31, Max flush: 61
34532024-02-20T22:43:15.266Z
34542024-02-20T22:43:15.266ZBegin loop 16
34552024-02-20T22:43:15.266ZDownstairs to restart: 1
34562024-02-20T22:43:15.279Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816158193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263}
34572024-02-20T22:43:15.292Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816501911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263Crucible runtime is spawned
34582024-02-20T22:43:15.292Z}
34592024-02-20T22:43:15.292Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816552259Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263}
34602024-02-20T22:43:15.292Z{"msg":"Crucible 44536089-83dd-40cd-98a8-7f3bb88f9370 has session id: 822e8cfc-7e19-4e5d-b4c1-c2183d377cae","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816582908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34612024-02-20T22:43:15.293Z{"msg":"Upstairs opts: Upstairs UUID: 44536089-83dd-40cd-98a8-7f3bb88f9370, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816646885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34622024-02-20T22:43:15.293Z{"msg":"Crucible stats registered with UUID: 44536089-83dd-40cd-98a8-7f3bb88f9370","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816685394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34632024-02-20T22:43:15.293Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816711103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263}
34642024-02-20T22:43:15.293Z{"msg":"44536089-83dd-40cd-98a8-7f3bb88f9370 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816754382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34652024-02-20T22:43:15.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.81679818Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34662024-02-20T22:43:15.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816834149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34672024-02-20T22:43:15.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816863078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34682024-02-20T22:43:15.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816883397Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34692024-02-20T22:43:15.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816908006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34702024-02-20T22:43:15.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816941535Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34712024-02-20T22:43:15.293Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816968864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34722024-02-20T22:43:15.293Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.816992023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34732024-02-20T22:43:15.293Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817026352Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34742024-02-20T22:43:15.293Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817089919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34752024-02-20T22:43:15.293Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817281972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34762024-02-20T22:43:15.293Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817325401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34772024-02-20T22:43:15.293Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817763145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34782024-02-20T22:43:15.293Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817811074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34792024-02-20T22:43:15.293Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817840603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34802024-02-20T22:43:15.293Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817881731Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34812024-02-20T22:43:15.293Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.81792486Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34822024-02-20T22:43:15.293Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.817956579Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34832024-02-20T22:43:15.293Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.818416862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34842024-02-20T22:43:15.293Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.818562397Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34852024-02-20T22:43:15.293Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.818620245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34862024-02-20T22:43:15.294Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.818882785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34872024-02-20T22:43:15.294Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.818918744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34882024-02-20T22:43:15.294Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.818934913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34892024-02-20T22:43:15.294Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819167345Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34902024-02-20T22:43:15.294Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819209444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34912024-02-20T22:43:15.294Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819248532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34922024-02-20T22:43:15.294Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819282381Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34932024-02-20T22:43:15.294Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.8193155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34942024-02-20T22:43:15.294Z{"msg":"[0]R flush_numbers[0..12]: [59, 58, 60, 50, 58, 48, 58, 61, 48, 60, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819348949Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34952024-02-20T22:43:15.294Z{"msg":"[0]R generation[0..12]: [30, 30, 30, 26, 30, 24, 30, 31, 24, 30, 26, 27]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
34962024-02-20T22:43:15.294Z,"time":"2024-02-20T22:43:14.819381708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34972024-02-20T22:43:15.294Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819419096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34982024-02-20T22:43:15.294Z{"msg":"[1]R flush_numbers[0..12]: [59, 58, 60, 50, 58, 48, 58, 61, 48, 60, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819451805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
34992024-02-20T22:43:15.294Z{"msg":"[1]R generation[0..12]: [30, 30, 30, 26, 30, 24, 30, 31, 24, 30, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819471624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35002024-02-20T22:43:15.294Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819501823Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35012024-02-20T22:43:15.294Z{"msg":"[2]R flush_numbers[0..12]: [59, 58, 60, 50, 58, 48, 58, 61, 48, 60, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819523882Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35022024-02-20T22:43:15.294Z{"msg":"[2]R generation[0..12]: [30, 30, 30, 26, 30, 24, 30, 31, 24, 30, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819552411Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35032024-02-20T22:43:15.294Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819573681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35042024-02-20T22:43:15.294Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.81960216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35052024-02-20T22:43:15.294Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819620889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35062024-02-20T22:43:15.294Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819650468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35072024-02-20T22:43:15.294Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819665767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35082024-02-20T22:43:15.294Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819696816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35092024-02-20T22:43:15.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35102024-02-20T22:43:15.295Z,"time":"2024-02-20T22:43:14.819715656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35112024-02-20T22:43:15.295Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819745395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35122024-02-20T22:43:15.295Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819776293Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35132024-02-20T22:43:15.295Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819806802Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35142024-02-20T22:43:15.295Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819838761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35152024-02-20T22:43:15.295Z{"msg":"44536089-83dd-40cd-98a8-7f3bb88f9370 is now active with session: 822e8cfc-7e19-4e5d-b4c1-c2183d377cae","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.8198711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35162024-02-20T22:43:15.295Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819902189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263,"session_id":"822e8cfc-7e19-4e5d-b4c1-c2183d377cae"}
35172024-02-20T22:43:15.295Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:14.819931578Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263}
35182024-02-20T22:43:15.295ZLoading write count information from file "/tmp/test_repair_verify.data"
35192024-02-20T22:43:15.295ZRun Repair workload
35202024-02-20T22:43:15.295Z01/30 Write block 338 len 24576 data: 4 4 3 2 3 3
35212024-02-20T22:43:15.300Z02/30 Read block 273 len 16384
35222024-02-20T22:43:15.300Z03/30 Read block 344 len 4096
35232024-02-20T22:43:15.300Z04/30 Read block 450 len 36864
35242024-02-20T22:43:15.300Z05/30 Write block 294 len 4096 data: 2
35252024-02-20T22:43:15.300Z06/30 Write block 339 len 40960 data: 5 4 3 4 4 3 3 3 5 5
35262024-02-20T22:43:15.300Z07/30 Read block 276 len 36864
35272024-02-20T22:43:15.300Z08/30 Read block 22 len 36864
35282024-02-20T22:43:15.300Z09/30 Write block 171 len 40960 data: 4 4 4 3 4 4 4 4 6 7
35292024-02-20T22:43:15.300Z10/30 Write block 306 len 32768 data: 3 4 3 4 4 4 4 4
35302024-02-20T22:43:15.300Z11/30 Read block 411 len 28672
35312024-02-20T22:43:15.301Z12/30 Read block 19 len 32768
35322024-02-20T22:43:15.301Z13/30 Read block 172 len 20480
35332024-02-20T22:43:15.301Z14/30 Write block 348 len 36864 data: 6 5 5 4 3 3 3 4 4
35342024-02-20T22:43:15.301Z15/30 Read block 286 len 36864
35352024-02-20T22:43:15.301Z16/30 Read block 289 len 36864
35362024-02-20T22:43:15.305Z17/30 Write block 530 len 28672 data: 5 4 2 2 3 4 3
35372024-02-20T22:43:15.305Z18/30 Write block 453 len 28672 data: 5 5 5 5 5 5 4
35382024-02-20T22:43:15.305Z19/30 Write block 301 len 24576 data: 4 4 4 3 3 4
35392024-02-20T22:43:15.306Z20/30 Read block 384 len 12288
35402024-02-20T22:43:15.306Z21/30 Write block 404 len 4096 data: 4
35412024-02-20T22:43:15.306Z22/30 Write block 510 len 8192 data: 4 4
35422024-02-20T22:43:15.306Z23/30 Write block 392 len 24576 data: 5 5 5 5 4 5
35432024-02-20T22:43:15.306Z24/30 Write block 55 len 4096 data: 3
35442024-02-20T22:43:15.311Z25/30 Write block 127 len 36864 data: 4 3 3 2 2 2 2 3 5
35452024-02-20T22:43:15.311Z26/30 Write block 15 len 16384 data: 3 4 6 4
35462024-02-20T22:43:15.311Z27/30 Write block 141 len 12288 data: 3 4 5
35472024-02-20T22:43:15.311Z28/30 Read block 369 len 20480
35482024-02-20T22:43:15.311Z29/30 Read block 21 len 32768
35492024-02-20T22:43:15.311Z30/30 Write block 27 len 12288 data: 6 7 7
35502024-02-20T22:43:15.312Z----------------------------------------------------------------
35512024-02-20T22:43:15.312Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35522024-02-20T22:43:15.312ZStates: Active Active Active
35532024-02-20T22:43:15.312ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35542024-02-20T22:43:15.312Z 1 Acked 1000 Write 6 Done Done Done false
35552024-02-20T22:43:15.312Z 2 Acked 1001 Read 4 Done Done Done false
35562024-02-20T22:43:15.312Z 3 Acked 1002 Read 1 Done Done Done false
35572024-02-20T22:43:15.312Z 4 Acked 1003 Read 9 Done Done Done false
35582024-02-20T22:43:15.312Z 5 Acked 1004 Write 1 Done Done Done false
35592024-02-20T22:43:15.312Z 6 Acked 1005 Write 10 Done Sent Done false
35602024-02-20T22:43:15.312Z 7 Acked 1006 Read 9 Done Sent Done false
35612024-02-20T22:43:15.312Z 8 Acked 1007 Read 9 Done Sent Done false
35622024-02-20T22:43:15.312Z 9 Acked 1008 Write 10 Done Sent Done false
35632024-02-20T22:43:15.312Z 10 Acked 1009 Write 8 Done Sent Done false
35642024-02-20T22:43:15.312Z 11 Acked 1010 Read 7 Done Sent Done false
35652024-02-20T22:43:15.312Z 12 Acked 1011 Read 8 Done Sent Done false
35662024-02-20T22:43:15.312Z 13 Acked 1012 Read 5 Done Sent Done false
35672024-02-20T22:43:15.312Z 14 Acked 1013 Write 9 Done Sent Done false
35682024-02-20T22:43:15.312Z 15 Acked 1014 Read 9 Done Sent Done false
35692024-02-20T22:43:15.312Z 16 Acked 1015 Read 9 Done Sent Done false
35702024-02-20T22:43:15.312Z 17 Acked 1016 Write 7 Done Sent Done false
35712024-02-20T22:43:15.312Z 18 Acked 1017 Write 7 Done Sent Done false
35722024-02-20T22:43:15.312Z 19 Acked 1018 Write 6 Done Sent Done false
35732024-02-20T22:43:15.312Z 20 Acked 1019 Read 3 Done Sent Done false
35742024-02-20T22:43:15.312Z 21 Acked 1020 Write 1 Done Sent Done false
35752024-02-20T22:43:15.313Z 22 Acked 1021 Write 2 Done Sent Done false
35762024-02-20T22:43:15.313Z 23 Acked 1022 Write 6 Done Sent Done false
35772024-02-20T22:43:15.313Z 24 Acked 1023 Write 1 Done Sent Done false
35782024-02-20T22:43:15.313Z 25 Acked 1024 Write 9 Done Sent Done false
35792024-02-20T22:43:15.313Z 26 Acked 1025 Write 4 Done Sent Done false
35802024-02-20T22:43:15.313Z 27 Acked 1026 Write 3 Done Sent Done false
35812024-02-20T22:43:15.313Z 28 Acked 1027 Read 5 Done Sent Done false
35822024-02-20T22:43:15.313Z 29 Acked 1028 Read 8 Done Sent Done false
35832024-02-20T22:43:15.313Z 30 Acked 1029 Write 3 Sent Sent Sent false
35842024-02-20T22:43:15.313Z STATES DS:0 DS:1 DS:2 TOTAL
35852024-02-20T22:43:15.313Z New 0 0 0 0
35862024-02-20T22:43:15.313Z Sent 1 25 1 27
35872024-02-20T22:43:15.313Z Done 29 5 29 63
35882024-02-20T22:43:15.313Z Skipped 0 0 0 0
35892024-02-20T22:43:15.313Z Error 0 0 0 0
35902024-02-20T22:43:15.313ZLast Flush: 0 0 0
35912024-02-20T22:43:15.313ZDownstairs last five completed:
35922024-02-20T22:43:15.313ZUpstairs last five completed: 30 29 28 27 26
35932024-02-20T22:43:15.313Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:14.847000379Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1263}
35942024-02-20T22:43:15.318ZWrote out file "/tmp/test_repair_verify.data"
35952024-02-20T22:43:15.318Z
35962024-02-20T22:43:15.318ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35972024-02-20T22:43:15.338ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35982024-02-20T22:43:15.338Z 0 000-019 30 30 30 59 59 59 T F T <---
35992024-02-20T22:43:15.338Z 1 020-039 30 30 30 58 58 58 T F T <---
36002024-02-20T22:43:15.338Z 2 040-059 30 30 30 60 60 60 T F T <---
36012024-02-20T22:43:15.338Z 3 060-079 26 26 26 50 50 50 F F F
36022024-02-20T22:43:15.338Z 4 080-099 30 30 30 58 58 58 F F F
36032024-02-20T22:43:15.338Z 5 100-119 24 24 24 48 48 48 F F F
36042024-02-20T22:43:15.338Z 6 120-139 30 30 30 58 58 58 T F T <---
36052024-02-20T22:43:15.338Z 7 140-159 31 31 31 61 61 61 T F T <---
36062024-02-20T22:43:15.338Z 8 160-179 24 24 24 48 48 48 T F T <---
36072024-02-20T22:43:15.338Z 9 180-199 30 30 30 60 60 60 T F T <---
36082024-02-20T22:43:15.338Z 10 200-219 26 26 26 50 50 50 F F F
36092024-02-20T22:43:15.338Z 11 220-239 27 27 27 51 51 51 F F F
36102024-02-20T22:43:15.338Z 12 240-259 27 27 27 51 51 51 F F F
36112024-02-20T22:43:15.338Z 13 260-279 26 26 26 50 50 50 F F F
36122024-02-20T22:43:15.343Z 14 280-299 27 27 27 51 51 51 T T T
36132024-02-20T22:43:15.343Z 15 300-319 27 27 27 51 51 51 T F T <---
36142024-02-20T22:43:15.343Z 16 320-339 29 29 29 57 57 57 T T T
36152024-02-20T22:43:15.343Z 17 340-359 30 30 30 59 59 59 T T T
36162024-02-20T22:43:15.343Z 18 360-379 30 30 30 58 58 58 F F F
36172024-02-20T22:43:15.343Z 19 380-399 23 23 23 45 45 45 T F T <---
36182024-02-20T22:43:15.343Z 20 400-419 31 31 31 61 61 61 T F T <---
36192024-02-20T22:43:15.343Z 21 420-439 19 19 19 38 38 38 F F F
36202024-02-20T22:43:15.343Z 22 440-459 30 30 30 58 58 58 T F T <---
36212024-02-20T22:43:15.344Z 23 460-479 28 28 28 55 55 55 F F F
36222024-02-20T22:43:15.344Z 24 480-499 31 31 31 61 61 61 F F F
36232024-02-20T22:43:15.344Z 25 500-519 31 31 31 61 61 61 T F T <---
36242024-02-20T22:43:15.344Z 26 520-539 19 19 19 38 38 38 T F T <---
36252024-02-20T22:43:15.344Z 27 540-559 23 23 23 45 45 45 F F F
36262024-02-20T22:43:15.344Z 28 560-579 12 12 12 23 23 23 F F F
36272024-02-20T22:43:15.344Z 29 580-599 25 25 25 49 49 49 F F F
36282024-02-20T22:43:15.344ZMax gen: 31, Max flush: 61
36292024-02-20T22:43:15.344ZError: Difference in extent metadata found!
36302024-02-20T22:43:15.344ZA Difference in extent metadata is expected here
36312024-02-20T22:43:15.344ZOn loop 16
36322024-02-20T22:43:15.344Z
36332024-02-20T22:43:15.348ZVerifying data now
36342024-02-20T22:43:21.143ZNo change after verify
36352024-02-20T22:43:21.144ZLoop: 16 Downstairs dump after verify (and repair):
36362024-02-20T22:43:21.158ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36372024-02-20T22:43:21.158Z 0 000-019 32 32 32 62 62 62 F F F
36382024-02-20T22:43:21.158Z 1 020-039 32 32 32 62 62 62 F F F
36392024-02-20T22:43:21.158Z 2 040-059 32 32 32 62 62 62 F F F
36402024-02-20T22:43:21.158Z 3 060-079 26 26 26 50 50 50 F F F
36412024-02-20T22:43:21.158Z 4 080-099 30 30 30 58 58 58 F F F
36422024-02-20T22:43:21.158Z 5 100-119 24 24 24 48 48 48 F F F
36432024-02-20T22:43:21.159Z 6 120-139 32 32 32 62 62 62 F F F
36442024-02-20T22:43:21.159Z 7 140-159 32 32 32 62 62 62 F F F
36452024-02-20T22:43:21.159Z 8 160-179 32 32 32 62 62 62 F F F
36462024-02-20T22:43:21.159Z 9 180-199 32 32 32 62 62 62 F F F
36472024-02-20T22:43:21.159Z 10 200-219 26 26 26 50 50 50 F F F
36482024-02-20T22:43:21.159Z 11 220-239 27 27 27 51 51 51 F F F
36492024-02-20T22:43:21.163Z 12 240-259 27 27 27 51 51 51 F F F
36502024-02-20T22:43:21.163Z 13 260-279 26 26 26 50 50 50 F F F
36512024-02-20T22:43:21.163Z 14 280-299 32 32 32 62 62 62 F F F
36522024-02-20T22:43:21.163Z 15 300-319 32 32 32 62 62 62 F F F
36532024-02-20T22:43:21.163Z 16 320-339 32 32 32 62 62 62 F F F
36542024-02-20T22:43:21.163Z 17 340-359 32 32 32 62 62 62 F F F
36552024-02-20T22:43:21.164Z 18 360-379 30 30 30 58 58 58 F F F
36562024-02-20T22:43:21.164Z 19 380-399 32 32 32 62 62 62 F F F
36572024-02-20T22:43:21.164Z 20 400-419 32 32 32 62 62 62 F F F
36582024-02-20T22:43:21.164Z 21 420-439 19 19 19 38 38 38 F F F
36592024-02-20T22:43:21.164Z 22 440-459 32 32 32 62 62 62 F F F
36602024-02-20T22:43:21.164Z 23 460-479 28 28 28 55 55 55 F F F
36612024-02-20T22:43:21.164Z 24 480-499 31 31 31 61 61 61 F F F
36622024-02-20T22:43:21.164Z 25 500-519 32 32 32 62 62 62 F F F
36632024-02-20T22:43:21.164Z 26 520-539 32 32 32 62 62 62 F F F
36642024-02-20T22:43:21.164Z 27 540-559 23 23 23 45 45 45 F F F
36652024-02-20T22:43:21.164Z 28 560-579 12 12 12 23 23 23 F F F
36662024-02-20T22:43:21.164Z 29 580-599 25 25 25 49 49 49 F F F
36672024-02-20T22:43:21.165ZMax gen: 32, Max flush: 62
36682024-02-20T22:43:21.165Z
36692024-02-20T22:43:21.165ZBegin loop 17
36702024-02-20T22:43:21.165ZDownstairs to restart: 2
36712024-02-20T22:43:21.180Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716230282Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271}
36722024-02-20T22:43:21.189Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716580659Z"Crucible runtime is spawned
36732024-02-20T22:43:21.189Z,"hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271}
36742024-02-20T22:43:21.189Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716624617Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271}
36752024-02-20T22:43:21.189Z{"msg":"Crucible 7f7414e5-c4b4-45fc-ad91-a1347a988272 has session id: c63a8608-120c-47ee-b632-448ee52349c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716657816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36762024-02-20T22:43:21.189Z{"msg":"Upstairs opts: Upstairs UUID: 7f7414e5-c4b4-45fc-ad91-a1347a988272, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716691935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36772024-02-20T22:43:21.189Z{"msg":"Crucible stats registered with UUID: 7f7414e5-c4b4-45fc-ad91-a1347a988272","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716719884Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36782024-02-20T22:43:21.189Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716733393Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271}
36792024-02-20T22:43:21.189Z{"msg":"7f7414e5-c4b4-45fc-ad91-a1347a988272 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716794711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36802024-02-20T22:43:21.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.7168218Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36812024-02-20T22:43:21.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.71683777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36822024-02-20T22:43:21.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716851869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36832024-02-20T22:43:21.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716866249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36842024-02-20T22:43:21.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716887278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36852024-02-20T22:43:21.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716901258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36862024-02-20T22:43:21.189Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716921257Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36872024-02-20T22:43:21.189Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716940856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36882024-02-20T22:43:21.189Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.716963205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36892024-02-20T22:43:21.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717127209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36902024-02-20T22:43:21.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717163898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36912024-02-20T22:43:21.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717190517Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36922024-02-20T22:43:21.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717834844Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36932024-02-20T22:43:21.190Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717876403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36942024-02-20T22:43:21.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717904331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36952024-02-20T22:43:21.190Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.717930931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36962024-02-20T22:43:21.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.71796016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36972024-02-20T22:43:21.190Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.718041337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36982024-02-20T22:43:21.190Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.7185166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
36992024-02-20T22:43:21.190Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.718570768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37002024-02-20T22:43:21.190Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.718653335Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37012024-02-20T22:43:21.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719048011Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37022024-02-20T22:43:21.190Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719095419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37032024-02-20T22:43:21.190Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719136387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37042024-02-20T22:43:21.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719157887Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37052024-02-20T22:43:21.190Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719180816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37062024-02-20T22:43:21.190Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719195065Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37072024-02-20T22:43:21.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719214175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37082024-02-20T22:43:21.190Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719228934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37092024-02-20T22:43:21.190Z{"msg":"[0]R flush_numbers[0..12]: [62, 62, 62, 50, 58, 48, 62, 62, 62, 62, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719249983Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37102024-02-20T22:43:21.190Z{"msg":"[0]R generation[0..12]: [32, 32, 32, 26, 30, 24, 32, 32, 32, 32, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719264483Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37112024-02-20T22:43:21.190Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719285382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
37122024-02-20T22:43:21.190Zsession_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37132024-02-20T22:43:21.190Z{"msg":"[1]R flush_numbers[0..12]: [62, 62, 62, 50, 58, 48, 62, 62, 62, 62, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719303572Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37142024-02-20T22:43:21.190Z{"msg":"[1]R generation[0..12]: [32, 32, 32, 26, 30, 24, 32, 32, 32, 32, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.7193346Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37152024-02-20T22:43:21.190Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719364639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37162024-02-20T22:43:21.191Z{"msg":"[2]R flush_numbers[0..12]: [62, 62, 62, 50, 58, 48, 62, 62, 62, 62, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719380259Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37172024-02-20T22:43:21.191Z{"msg":"[2]R generation[0..12]: [32, 32, 32, 26, 30, 24, 32, 32, 32, 32, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719413957Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37182024-02-20T22:43:21.191Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719447786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37192024-02-20T22:43:21.191Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719476855Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37202024-02-20T22:43:21.191Z{"msg":"Generation requested: 34 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719495995Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37212024-02-20T22:43:21.191Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719527884Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37222024-02-20T22:43:21.191Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719543633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37232024-02-20T22:43:21.191Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719565382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37242024-02-20T22:43:21.191Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719577852Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37252024-02-20T22:43:21.191Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719592661Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37262024-02-20T22:43:21.191Zclient":"1","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37272024-02-20T22:43:21.191Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.71961832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37282024-02-20T22:43:21.191Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.71963203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37292024-02-20T22:43:21.191Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719651969Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37302024-02-20T22:43:21.191Z{"msg":"7f7414e5-c4b4-45fc-ad91-a1347a988272 is now active with session: c63a8608-120c-47ee-b632-448ee52349c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719665499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37312024-02-20T22:43:21.191Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719684238Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271,"session_id":"c63a8608-120c-47ee-b632-448ee52349c9"}
37322024-02-20T22:43:21.191Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:20.719697038Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271}
37332024-02-20T22:43:21.191ZLoading write count information from file "/tmp/test_repair_verify.data"
37342024-02-20T22:43:21.191ZRun Repair workload
37352024-02-20T22:43:21.196Z01/30 Write block 418 len 32768 data: 4 4 4 3 2 2 2 2
37362024-02-20T22:43:21.196Z02/30 Read block 12 len 4096
37372024-02-20T22:43:21.196Z03/30 Read block 246 len 16384
37382024-02-20T22:43:21.196Z04/30 Read block 213 len 40960
37392024-02-20T22:43:21.196Z05/30 Read block 146 len 40960
37402024-02-20T22:43:21.196Z06/30 Write block 370 len 36864 data: 6 7 7 4 4 4 4 5 4
37412024-02-20T22:43:21.196Z07/30 Read block 42 len 24576
37422024-02-20T22:43:21.197Z08/30 Flush
37432024-02-20T22:43:21.197Z09/30 Write block 157 len 40960 data: 3 3 3 3 3 2 2 2 4 4
37442024-02-20T22:43:21.197Z10/30 Read block 435 len 24576
37452024-02-20T22:43:21.197Z11/30 Read block 390 len 36864
37462024-02-20T22:43:21.201Z12/30 Read block 88 len 4096
37472024-02-20T22:43:21.201Z13/30 Write block 35 len 40960 data: 4 4 4 4 4 3 3 4 3 3
37482024-02-20T22:43:21.201Z14/30 Write block 502 len 32768 data: 2 2 3 3 3 3 3 3
37492024-02-20T22:43:21.201Z15/30 Read block 419 len 24576
37502024-02-20T22:43:21.201Z16/30 Read block 497 len 4096
37512024-02-20T22:43:21.204Z17/30 Write block 443 len 12288 data: 3 3 3
37522024-02-20T22:43:21.204Z18/30 Write block 315 len 32768 data: 5 4 4 3 3 4 4 5
37532024-02-20T22:43:21.204Z19/30 Write block 32 len 16384 data: 6 5 4 5
37542024-02-20T22:43:21.204Z20/30 Write block 493 len 40960 data: 4 4 4 5 6 5 5 5 4 3
37552024-02-20T22:43:21.205Z21/30 Write block 326 len 36864 data: 3 3 3 4 4 5 6 6 5
37562024-02-20T22:43:21.205Z22/30 Read block 496 len 16384
37572024-02-20T22:43:21.208Z23/30 Write block 27 len 32768 data: 7 8 8 8 7 7 6 5
37582024-02-20T22:43:21.208Z24/30 Read block 472 len 12288
37592024-02-20T22:43:21.208Z25/30 Write block 536 len 4096 data: 4
37602024-02-20T22:43:21.208Z26/30 Read block 466 len 4096
37612024-02-20T22:43:21.213Z27/30 Write block 110 len 20480 data: 5 6 7 8 6
37622024-02-20T22:43:21.213Z28/30 Write block 434 len 8192 data: 2 2
37632024-02-20T22:43:21.213Z29/30 Read block 398 len 28672
37642024-02-20T22:43:21.213Z30/30 Write block 325 len 24576 data: 3 4 4 4 5 5
37652024-02-20T22:43:21.213Z----------------------------------------------------------------
37662024-02-20T22:43:21.213Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37672024-02-20T22:43:21.213ZStates: Active Active Active
37682024-02-20T22:43:21.213ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37692024-02-20T22:43:21.213Z 1 Acked 1000 Write 8 Done Done Sent false
37702024-02-20T22:43:21.213Z 2 Acked 1001 Read 1 Done Done Sent false
37712024-02-20T22:43:21.214Z 3 Acked 1002 Read 4 Done Done Sent false
37722024-02-20T22:43:21.214Z 4 Acked 1003 Read 10 Done Done Sent false
37732024-02-20T22:43:21.214Z 5 Acked 1004 Read 10 Done Done Sent false
37742024-02-20T22:43:21.214Z 6 Acked 1005 Write 9 Done Done Sent false
37752024-02-20T22:43:21.214Z 7 Acked 1006 Read 6 Done Done Sent false
37762024-02-20T22:43:21.214Z 8 Acked 1007 Flush 0 Done Done Sent false
37772024-02-20T22:43:21.214Z 9 Acked 1008 Write 10 Done Done Sent false
37782024-02-20T22:43:21.214Z 10 Acked 1009 Read 6 Done Done Sent false
37792024-02-20T22:43:21.214Z 11 Acked 1010 Read 9 Done Done Sent false
37802024-02-20T22:43:21.214Z 12 Acked 1011 Read 1 Done Done Sent false
37812024-02-20T22:43:21.214Z 13 Acked 1012 Write 10 Done Done Sent false
37822024-02-20T22:43:21.214Z 14 Acked 1013 Write 8 Done Done Sent false
37832024-02-20T22:43:21.214Z 15 Acked 1014 Read 6 Done Done Sent false
37842024-02-20T22:43:21.215Z 16 Acked 1015 Read 1 Done Done Sent false
37852024-02-20T22:43:21.215Z 17 Acked 1016 Write 3 Done Done Sent false
37862024-02-20T22:43:21.215Z 18 Acked 1017 Write 8 Done Done Sent false
37872024-02-20T22:43:21.215Z 19 Acked 1018 Write 4 Done Done Sent false
37882024-02-20T22:43:21.215Z 20 Acked 1019 Write 10 Done Done Sent false
37892024-02-20T22:43:21.215Z 21 Acked 1020 Write 9 Done Done Sent false
37902024-02-20T22:43:21.215Z 22 Acked 1021 Read 4 Done Done Sent false
37912024-02-20T22:43:21.215Z 23 Acked 1022 Write 8 Done Done Sent false
37922024-02-20T22:43:21.215Z 24 Acked 1023 Read 3 Done Done Sent false
37932024-02-20T22:43:21.215Z 25 Acked 1024 Write 1 Done Done Sent false
37942024-02-20T22:43:21.215Z 26 Acked 1025 Read 1 Done Done Sent false
37952024-02-20T22:43:21.215Z 27 Acked 1026 Write 5 Done Done Sent false
37962024-02-20T22:43:21.215Z 28 Acked 1027 Write 2 Done Done Sent false
37972024-02-20T22:43:21.216Z 29 Acked 1028 Read 7 Done Done Sent false
37982024-02-20T22:43:21.216Z 30 Acked 1029 Write 6 Sent Sent Sent false
37992024-02-20T22:43:21.216Z STATES DS:0 DS:1 DS:2 TOTAL
38002024-02-20T22:43:21.216Z New 0 0 0 0
38012024-02-20T22:43:21.216Z Sent 1 1 30 32
38022024-02-20T22:43:21.216Z Done 29 29 0 58
38032024-02-20T22:43:21.216Z Skipped 0 0 0 0
38042024-02-20T22:43:21.216Z Error 0 0 0 0
38052024-02-20T22:43:21.216ZLast Flush: 1007 1007 0
38062024-02-20T22:43:21.216ZDownstairs last five completed:
38072024-02-20T22:43:21.216ZUpstairs last five completed: 30 29 28 27 26
38082024-02-20T22:43:21.216Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:20.748270645Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1271}
38092024-02-20T22:43:21.216ZWrote out file "/tmp/test_repair_verify.data"
38102024-02-20T22:43:21.221Z
38112024-02-20T22:43:21.221ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38122024-02-20T22:43:21.236ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38132024-02-20T22:43:21.236Z 0 000-019 32 32 32 62 62 62 F F F
38142024-02-20T22:43:21.236Z 1 020-039 32 32 32 62 62 62 T T F <---
38152024-02-20T22:43:21.236Z 2 040-059 32 32 32 62 62 62 T T F <---
38162024-02-20T22:43:21.237Z 3 060-079 26 26 26 50 50 50 F F F
38172024-02-20T22:43:21.237Z 4 080-099 30 30 30 58 58 58 F F F
38182024-02-20T22:43:21.237Z 5 100-119 24 24 24 48 48 48 T T F <---
38192024-02-20T22:43:21.237Z 6 120-139 32 32 32 62 62 62 F F F
38202024-02-20T22:43:21.240Z 7 140-159 32 32 32 62 62 62 T T F <---
38212024-02-20T22:43:21.240Z 8 160-179 32 32 32 62 62 62 T T F <---
38222024-02-20T22:43:21.240Z 9 180-199 32 32 32 62 62 62 F F F
38232024-02-20T22:43:21.241Z 10 200-219 26 26 26 50 50 50 F F F
38242024-02-20T22:43:21.241Z 11 220-239 27 27 27 51 51 51 F F F
38252024-02-20T22:43:21.241Z 12 240-259 27 27 27 51 51 51 F F F
38262024-02-20T22:43:21.241Z 13 260-279 26 26 26 50 50 50 F F F
38272024-02-20T22:43:21.241Z 14 280-299 32 32 32 62 62 62 F F F
38282024-02-20T22:43:21.241Z 15 300-319 32 32 32 62 62 62 T T F <---
38292024-02-20T22:43:21.241Z 16 320-339 32 32 32 62 62 62 T T F <---
38302024-02-20T22:43:21.241Z 17 340-359 32 32 32 62 62 62 F F F
38312024-02-20T22:43:21.241Z 18 360-379 34 34 30 63 63 58 F F F <---
38322024-02-20T22:43:21.241Z 19 380-399 32 32 32 62 62 62 F F F
38332024-02-20T22:43:21.241Z 20 400-419 34 34 32 63 63 62 F F F <---
38342024-02-20T22:43:21.241Z 21 420-439 34 34 19 63 63 38 T T F <---
38352024-02-20T22:43:21.241Z 22 440-459 32 32 32 62 62 62 T T F <---
38362024-02-20T22:43:21.241Z 23 460-479 28 28 28 55 55 55 F F F
38372024-02-20T22:43:21.241Z 24 480-499 31 31 31 61 61 61 T T F <---
38382024-02-20T22:43:21.241Z 25 500-519 32 32 32 62 62 62 T T F <---
38392024-02-20T22:43:21.241Z 26 520-539 32 32 32 62 62 62 T T F <---
38402024-02-20T22:43:21.241Z 27 540-559 23 23 23 45 45 45 F F F
38412024-02-20T22:43:21.241Z 28 560-579 12 12 12 23 23 23 F F F
38422024-02-20T22:43:21.241Z 29 580-599 25 25 25 49 49 49 F F F
38432024-02-20T22:43:21.241ZMax gen: 34, Max flush: 63
38442024-02-20T22:43:21.241ZError: Difference in extent metadata found!
38452024-02-20T22:43:21.241ZA Difference in extent metadata is expected here
38462024-02-20T22:43:21.241ZOn loop 17
38472024-02-20T22:43:21.242Z
38482024-02-20T22:43:21.246ZVerifying data now
38492024-02-20T22:43:25.931ZNo change after verify
38502024-02-20T22:43:25.931ZLoop: 17 Downstairs dump after verify (and repair):
38512024-02-20T22:43:25.955ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38522024-02-20T22:43:25.955Z 0 000-019 32 32 32 62 62 62 F F F
38532024-02-20T22:43:25.955Z 1 020-039 35 35 35 64 64 64 F F F
38542024-02-20T22:43:25.955Z 2 040-059 35 35 35 64 64 64 F F F
38552024-02-20T22:43:25.955Z 3 060-079 26 26 26 50 50 50 F F F
38562024-02-20T22:43:25.955Z 4 080-099 30 30 30 58 58 58 F F F
38572024-02-20T22:43:25.984Z 5 100-119 35 35 35 64 64 64 F F F
38582024-02-20T22:43:25.985Z 6 120-139 32 32 32 62 62 62 F F F
38592024-02-20T22:43:25.985Z 7 140-159 35 35 35 64 64 64 F F F
38602024-02-20T22:43:25.985Z 8 160-179 35 35 35 64 64 64 F F F
38612024-02-20T22:43:25.985Z 9 180-199 32 32 32 62 62 62 F F F
38622024-02-20T22:43:25.985Z 10 200-219 26 26 26 50 50 50 F F F
38632024-02-20T22:43:25.985Z 11 220-239 27 27 27 51 51 51 F F F
38642024-02-20T22:43:25.985Z 12 240-259 27 27 27 51 51 51 F F F
38652024-02-20T22:43:25.985Z 13 260-279 26 26 26 50 50 50 F F F
38662024-02-20T22:43:25.985Z 14 280-299 32 32 32 62 62 62 F F F
38672024-02-20T22:43:25.985Z 15 300-319 35 35 35 64 64 64 F F F
38682024-02-20T22:43:25.985Z 16 320-339 35 35 35 64 64 64 F F F
38692024-02-20T22:43:25.985Z 17 340-359 32 32 32 62 62 62 F F F
38702024-02-20T22:43:25.985Z 18 360-379 34 34 34 63 63 63 F F F
38712024-02-20T22:43:25.985Z 19 380-399 32 32 32 62 62 62 F F F
38722024-02-20T22:43:25.985Z 20 400-419 34 34 34 63 63 63 F F F
38732024-02-20T22:43:25.985Z 21 420-439 35 35 35 64 64 64 F F F
38742024-02-20T22:43:25.985Z 22 440-459 35 35 35 64 64 64 F F F
38752024-02-20T22:43:25.985Z 23 460-479 28 28 28 55 55 55 F F F
38762024-02-20T22:43:25.985Z 24 480-499 35 35 35 64 64 64 F F F
38772024-02-20T22:43:25.985Z 25 500-519 35 35 35 64 64 64 F F F
38782024-02-20T22:43:25.985Z 26 520-539 35 35 35 64 64 64 F F F
38792024-02-20T22:43:25.985Z 27 540-559 23 23 23 45 45 45 F F F
38802024-02-20T22:43:25.985Z 28 560-579 12 12 12 23 23 23 F F F
38812024-02-20T22:43:25.985Z 29 580-599 25 25 25 49 49 49 F F F
38822024-02-20T22:43:25.986ZMax gen: 35, Max flush: 64
38832024-02-20T22:43:25.986Z
38842024-02-20T22:43:25.986ZBegin loop 18
38852024-02-20T22:43:25.986ZDownstairs to restart: 2
38862024-02-20T22:43:25.986Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496071355Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279}
38872024-02-20T22:43:25.986Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496413893Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279}
38882024-02-20T22:43:25.986Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
38892024-02-20T22:43:25.986Z2024-02-20T22:43:25.496452271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279}
38902024-02-20T22:43:25.986Z{"msg":"Crucible 1166b40d-9828-4862-8b08-db83d1aa4204 has session id: a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.49649859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38912024-02-20T22:43:25.986Z{"msg":"Upstairs opts: Upstairs UUID: 1166b40d-9828-4862-8b08-db83d1aa4204, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496534648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38922024-02-20T22:43:25.986Z{"msg":"Crucible stats registered with UUID: 1166b40d-9828-4862-8b08-db83d1aa4204","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496569247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38932024-02-20T22:43:25.986Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496593136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279}
38942024-02-20T22:43:25.986Z{"msg":"1166b40d-9828-4862-8b08-db83d1aa4204 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496654384Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38952024-02-20T22:43:25.986Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496706692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38962024-02-20T22:43:25.986Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496744361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38972024-02-20T22:43:25.986Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.49677837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38982024-02-20T22:43:25.986Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496827128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
38992024-02-20T22:43:25.986Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496863786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39002024-02-20T22:43:25.986Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.496973563Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39012024-02-20T22:43:25.986Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.49703643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39022024-02-20T22:43:25.986Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497099728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39032024-02-20T22:43:25.986Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497138367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39042024-02-20T22:43:25.986Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497188515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39052024-02-20T22:43:25.986Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497213724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39062024-02-20T22:43:25.987Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497251973Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39072024-02-20T22:43:25.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497689387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39082024-02-20T22:43:25.987Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497725636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39092024-02-20T22:43:25.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497741755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39102024-02-20T22:43:25.987Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497756455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39112024-02-20T22:43:25.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497782264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39122024-02-20T22:43:25.987Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.497806263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39132024-02-20T22:43:26.023Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.498336544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39142024-02-20T22:43:26.023Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.498377072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39152024-02-20T22:43:26.023Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.498631343Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39162024-02-20T22:43:26.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.498874395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39172024-02-20T22:43:26.023Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.498939972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39182024-02-20T22:43:26.023Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.498955292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39192024-02-20T22:43:26.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499024639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39202024-02-20T22:43:26.023Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499052618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39212024-02-20T22:43:26.024Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499067308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39222024-02-20T22:43:26.024Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499143325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39232024-02-20T22:43:26.024Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499180284Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39242024-02-20T22:43:26.024Z{"msg":"[0]R flush_numbers[0..12]: [62, 64, 64, 50, 58, 64, 62, 64, 64, 62, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499196993Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39252024-02-20T22:43:26.024Z{"msg":"[0]R generation[0..12]: [32, 35, 35, 26, 30, 35, 32, 35, 35, 32, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499218422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39262024-02-20T22:43:26.024Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499233582Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39272024-02-20T22:43:26.024Z{"msg":"[1]R flush_numbers[0..12]: [62, 64, 64, 50, 58, 64, 62, 64, 64, 62, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499255461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39282024-02-20T22:43:26.024Z{"msg":"[1]R generation[0..12]: [32, 35, 35, 26, 30, 35, 32, 35, 35, 32, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.49926984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39292024-02-20T22:43:26.024Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.49929062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39302024-02-20T22:43:26.024Z{"msg":"[2]R flush_numbers[0..12]: [62, 64, 64, 50, 58, 64, 62, 64, 64, 62, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499304889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39312024-02-20T22:43:26.024Z{"msg":"[2]R generation[0..12]: [32, 35, 35, 26, 30, 35, 32, 35, 35, 32, 26, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499325109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39322024-02-20T22:43:26.024Z{"msg":"Wait for a query_work_queue command to finish before sending IO
39332024-02-20T22:43:26.024Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499342458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39342024-02-20T22:43:26.024Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499357437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39352024-02-20T22:43:26.024Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499371087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39362024-02-20T22:43:26.024Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499390746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39372024-02-20T22:43:26.024Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499404236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39382024-02-20T22:43:26.024Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499427465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39392024-02-20T22:43:26.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499439624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39402024-02-20T22:43:26.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499468333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39412024-02-20T22:43:26.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499491432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39422024-02-20T22:43:26.024Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499511372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39432024-02-20T22:43:26.024Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499528591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39442024-02-20T22:43:26.024Z{"msg":"1166b40d-9828-4862-8b08-db83d1aa4204 is now active with session: a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.49956021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39452024-02-20T22:43:26.025Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499581749Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279,"session_id":"a9ec118a-8bbc-4a28-8ef8-a0e40d70eeff"}
39462024-02-20T22:43:26.025Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:25.499600859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279}
39472024-02-20T22:43:26.025ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39482024-02-20T22:43:26.025ZLoading write count information from file "/tmp/test_repair_verify.data"
39492024-02-20T22:43:26.025ZRun Repair workload
39502024-02-20T22:43:26.025Z01/30 Write block 465 len 20480 data: 8 8 8 7 7
39512024-02-20T22:43:26.025Z02/30 Write block 426 len 24576 data: 2 2 2 2 2 2
39522024-02-20T22:43:26.025Z03/30 Write block 547 len 12288 data: 3 3 4
39532024-02-20T22:43:26.025Z04/30 Read block 238 len 16384
39542024-02-20T22:43:26.025Z05/30 Read block 432 len 16384
39552024-02-20T22:43:26.025Z06/30 Read block 442 len 36864
39562024-02-20T22:43:26.025Z07/30 Write block 247 len 16384 data: 2 2 2 2
39572024-02-20T22:43:26.025Z08/30 Flush
39582024-02-20T22:43:26.025Z09/30 Write block 319 len 40960 data: 4 5 5 6 3 2 4 5 5 5
39592024-02-20T22:43:26.025Z10/30 Read block 385 len 16384
39602024-02-20T22:43:26.025Z11/30 Write block 397 len 36864 data: 6 4 4 4 4 5 5 5 5
39612024-02-20T22:43:26.025Z12/30 Read block 90 len 40960
39622024-02-20T22:43:26.025Z13/30 Write block 378 len 12288 data: 5 3 2
39632024-02-20T22:43:26.025Z14/30 Write block 283 len 40960 data: 4 4 3 4 3 2 2 2 2 3
39642024-02-20T22:43:26.025Z15/30 Read block 390 len 8192
39652024-02-20T22:43:26.025Z16/30 Write block 438 len 40960 data: 2 3 3 3 3 4 4 4 3 3
39662024-02-20T22:43:26.025Z17/30 Read block 444 len 12288
39672024-02-20T22:43:26.025Z18/30 Write block 264 len 24576 data: 4 4 4 4 5 5
39682024-02-20T22:43:26.025Z19/30 Read block 498 len 12288
39692024-02-20T22:43:26.025Z20/30 Read block 184 len 12288
39702024-02-20T22:43:26.025Z21/30 Write block 182 len 40960 data: 6 5 6 5 4 4 4 4 3 2
39712024-02-20T22:43:26.026Z22/30 Write block 576 len 16384 data: 4 4 4 4
39722024-02-20T22:43:26.026Z23/30 Read block 35 len 4096
39732024-02-20T22:43:26.026Z24/30 Read block 331 len 32768
39742024-02-20T22:43:26.026Z25/30 Read block 187 len 8192
39752024-02-20T22:43:26.026Z26/30 Write block 79 len 8192 data: 3 3
39762024-02-20T22:43:26.026Z27/30 Write block 228 len 28672 data: 3 4 5 4 5 5 5
39772024-02-20T22:43:26.026Z28/30 Write block 119 len 20480 data: 4 4 3 4 3
39782024-02-20T22:43:26.026Z29/30 Write block 415 len 8192 data: 6 6
39792024-02-20T22:43:26.026Z30/30 Read block 228 len 32768
39802024-02-20T22:43:26.026Z----------------------------------------------------------------
39812024-02-20T22:43:26.026Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39822024-02-20T22:43:26.026ZStates: Active Active Active
39832024-02-20T22:43:26.026ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39842024-02-20T22:43:26.026Z 1 Acked 1000 Write 5 Done Done Sent false
39852024-02-20T22:43:26.026Z 2 Acked 1001 Write 6 Done Done Sent false
39862024-02-20T22:43:26.026Z 3 Acked 1002 Write 3 Done Done Sent false
39872024-02-20T22:43:26.026Z 4 Acked 1003 Read 4 Done Done Sent false
39882024-02-20T22:43:26.026Z 5 Acked 1004 Read 4 Done Done Sent false
39892024-02-20T22:43:26.026Z 6 Acked 1005 Read 9 Done Done Sent false
39902024-02-20T22:43:26.026Z 7 Acked 1006 Write 4 Done Done Sent false
39912024-02-20T22:43:26.026Z 8 Acked 1007 Flush 0 Done Done Sent false
39922024-02-20T22:43:26.026Z 9 Acked 1008 Write 10 Done Done Sent false
39932024-02-20T22:43:26.026Z 10 Acked 1009 Read 4 Done Done Sent false
39942024-02-20T22:43:26.026Z 11 Acked 1010 Write 9 Done Done Sent false
39952024-02-20T22:43:26.026Z 12 Acked 1011 Read 10 Done Done Sent false
39962024-02-20T22:43:26.026Z 13 Acked 1012 Write 3 Done Done Sent false
39972024-02-20T22:43:26.027Z 14 Acked 1013 Write 10 Done Done Sent false
39982024-02-20T22:43:26.027Z 15 Acked 1014 Read 2 Done Done Sent false
39992024-02-20T22:43:26.027Z 16 Acked 1015 Write 10 Done Done Sent false
40002024-02-20T22:43:26.027Z 17 Acked 1016 Read 3 Done Done Sent false
40012024-02-20T22:43:26.027Z 18 Acked 1017 Write 6 Done Done Sent false
40022024-02-20T22:43:26.027Z 19 Acked 1018 Read 3 Done Done Sent false
40032024-02-20T22:43:26.027Z 20 Acked 1019 Read 3 Done Done Sent false
40042024-02-20T22:43:26.027Z 21 Acked 1020 Write 10 Done Done Sent false
40052024-02-20T22:43:26.027Z 22 Acked 1021 Write 4 Done Done Sent false
40062024-02-20T22:43:26.027Z 23 Acked 1022 Read 1 Done Done Sent false
40072024-02-20T22:43:26.027Z 24 Acked 1023 Read 8 Done Done Sent false
40082024-02-20T22:43:26.027Z 25 Acked 1024 Read 2 Done Done Sent false
40092024-02-20T22:43:26.027Z 26 Acked 1025 Write 2 Done Done Sent false
40102024-02-20T22:43:26.027Z 27 Acked 1026 Write 7 Done Done Sent false
40112024-02-20T22:43:26.027Z 28 Acked 1027 Write 5 Done Done Sent false
40122024-02-20T22:43:26.027Z 29 Acked 1028 Write 2 Done Done Sent false
40132024-02-20T22:43:26.042Z 30 Acked 1029 Read 8 Done Done Sent false
40142024-02-20T22:43:26.042Z STATES DS:0 DS:1 DS:2 TOTAL
40152024-02-20T22:43:26.042Z New 0 0 0 0
40162024-02-20T22:43:26.043Z Sent 0 0 30 30
40172024-02-20T22:43:26.043Z Done 30 30 0 60
40182024-02-20T22:43:26.043Z Skipped 0 0 0 0
40192024-02-20T22:43:26.043Z Error 0 0 0 0
40202024-02-20T22:43:26.043ZLast Flush: 1007 1007 0
40212024-02-20T22:43:26.043ZDownstairs last five completed:
40222024-02-20T22:43:26.043ZUpstairs last five completed: 30 29 28 27 26
40232024-02-20T22:43:26.043Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:25.528855522Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1279}
40242024-02-20T22:43:26.043ZWrote out file "/tmp/test_repair_verify.data"
40252024-02-20T22:43:26.043Z
40262024-02-20T22:43:26.043ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40272024-02-20T22:43:26.043ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40282024-02-20T22:43:26.043Z 0 000-019 32 32 32 62 62 62 F F F
40292024-02-20T22:43:26.043Z 1 020-039 35 35 35 64 64 64 F F F
40302024-02-20T22:43:26.043Z 2 040-059 35 35 35 64 64 64 F F F
40312024-02-20T22:43:26.043Z 3 060-079 26 26 26 50 50 50 T T F <---
40322024-02-20T22:43:26.043Z 4 080-099 30 30 30 58 58 58 T T F <---
40332024-02-20T22:43:26.043Z 5 100-119 35 35 35 64 64 64 T T F <---
40342024-02-20T22:43:26.043Z 6 120-139 32 32 32 62 62 62 T T F <---
40352024-02-20T22:43:26.043Z 7 140-159 35 35 35 64 64 64 F F F
40362024-02-20T22:43:26.043Z 8 160-179 35 35 35 64 64 64 F F F
40372024-02-20T22:43:26.043Z 9 180-199 32 32 32 62 62 62 T T F <---
40382024-02-20T22:43:26.043Z 10 200-219 26 26 26 50 50 50 F F F
40392024-02-20T22:43:26.044Z 11 220-239 27 27 27 51 51 51 T T F <---
40402024-02-20T22:43:26.044Z 12 240-259 36 36 27 65 65 51 F F F <---
40412024-02-20T22:43:26.044Z 13 260-279 26 26 26 50 50 50 T T F <---
40422024-02-20T22:43:26.044Z 14 280-299 32 32 32 62 62 62 T T F <---
40432024-02-20T22:43:26.044Z 15 300-319 35 35 35 64 64 64 T T F <---
40442024-02-20T22:43:26.044Z 16 320-339 35 35 35 64 64 64 T T F <---
40452024-02-20T22:43:26.044Z 17 340-359 32 32 32 62 62 62 F F F
40462024-02-20T22:43:26.044Z 18 360-379 34 34 34 63 63 63 T T F <---
40472024-02-20T22:43:26.044Z 19 380-399 32 32 32 62 62 62 T T F <---
40482024-02-20T22:43:26.044Z 20 400-419 34 34 34 63 63 63 T T F <---
40492024-02-20T22:43:26.044Z 21 420-439 36 36 35 65 65 64 T T F <---
40502024-02-20T22:43:26.044Z 22 440-459 35 35 35 64 64 64 T T F <---
40512024-02-20T22:43:26.044Z 23 460-479 36 36 28 65 65 55 F F F <---
40522024-02-20T22:43:26.044Z 24 480-499 35 35 35 64 64 64 F F F
40532024-02-20T22:43:26.044Z 25 500-519 35 35 35 64 64 64 F F F
40542024-02-20T22:43:26.044Z 26 520-539 35 35 35 64 64 64 F F F
40552024-02-20T22:43:26.044Z 27 540-559 36 36 23 65 65 45 F F F <---
40562024-02-20T22:43:26.044Z 28 560-579 12 12 12 23 23 23 T T F <---
40572024-02-20T22:43:26.044Z 29 580-599 25 25 25 49 49 49 F F F
40582024-02-20T22:43:26.044ZMax gen: 36, Max flush: 65
40592024-02-20T22:43:26.044ZError: Difference in extent metadata found!
40602024-02-20T22:43:26.044ZA Difference in extent metadata is expected here
40612024-02-20T22:43:26.044ZOn loop 18
40622024-02-20T22:43:26.044Z
40632024-02-20T22:43:26.044ZVerifying data now
40642024-02-20T22:43:30.935ZNo change after verify
40652024-02-20T22:43:30.935ZLoop: 18 Downstairs dump after verify (and repair):
40662024-02-20T22:43:30.942ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40672024-02-20T22:43:30.943Z 0 000-019 32 32 32 62 62 62 F F F
40682024-02-20T22:43:30.943Z 1 020-039 35 35 35 64 64 64 F F F
40692024-02-20T22:43:30.943Z 2 040-059 35 35 35 64 64 64 F F F
40702024-02-20T22:43:30.943Z 3 060-079 37 37 37 66 66 66 F F F
40712024-02-20T22:43:30.943Z 4 080-099 37 37 37 66 66 66 F F F
40722024-02-20T22:43:30.943Z 5 100-119 37 37 37 66 66 66 F F F
40732024-02-20T22:43:30.943Z 6 120-139 37 37 37 66 66 66 F F F
40742024-02-20T22:43:30.943Z 7 140-159 35 35 35 64 64 64 F F F
40752024-02-20T22:43:30.943Z 8 160-179 35 35 35 64 64 64 F F F
40762024-02-20T22:43:30.943Z 9 180-199 37 37 37 66 66 66 F F F
40772024-02-20T22:43:30.943Z 10 200-219 26 26 26 50 50 50 F F F
40782024-02-20T22:43:30.943Z 11 220-239 37 37 37 66 66 66 F F F
40792024-02-20T22:43:30.943Z 12 240-259 36 36 36 65 65 65 F F F
40802024-02-20T22:43:30.943Z 13 260-279 37 37 37 66 66 66 F F F
40812024-02-20T22:43:30.943Z 14 280-299 37 37 37 66 66 66 F F F
40822024-02-20T22:43:30.943Z 15 300-319 37 37 37 66 66 66 F F F
40832024-02-20T22:43:30.943Z 16 320-339 37 37 37 66 66 66 F F F
40842024-02-20T22:43:30.943Z 17 340-359 32 32 32 62 62 62 F F F
40852024-02-20T22:43:30.943Z 18 360-379 37 37 37 66 66 66 F F F
40862024-02-20T22:43:30.943Z 19 380-399 37 37 37 66 66 66 F F F
40872024-02-20T22:43:30.943Z 20 400-419 37 37 37 66 66 66 F F F
40882024-02-20T22:43:30.943Z 21 420-439 37 37 37 66 66 66 F F F
40892024-02-20T22:43:30.943Z 22 440-459 37 37 37 66 66 66 F F F
40902024-02-20T22:43:30.943Z 23 460-479 36 36 36 65 65 65 F F F
40912024-02-20T22:43:30.944Z 24 480-499 35 35 35 64 64 64 F F F
40922024-02-20T22:43:30.944Z 25 500-519 35 35 35 64 64 64 F F F
40932024-02-20T22:43:30.944Z 26 520-539 35 35 35 64 64 64 F F F
40942024-02-20T22:43:30.944Z 27 540-559 36 36 36 65 65 65 F F F
40952024-02-20T22:43:30.944Z 28 560-579 37 37 37 66 66 66 F F F
40962024-02-20T22:43:30.944Z 29 580-599 25 25 25 49 49 49 F F F
40972024-02-20T22:43:30.944ZMax gen: 37, Max flush: 66
40982024-02-20T22:43:30.956Z
40992024-02-20T22:43:30.956ZBegin loop 19
41002024-02-20T22:43:30.956ZDownstairs to restart: 2
41012024-02-20T22:43:30.965Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496140177Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287}
41022024-02-20T22:43:30.965Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496486715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287}
41032024-02-20T22:43:30.965ZCrucible runtime is spawned
41042024-02-20T22:43:30.965Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496539023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287}
41052024-02-20T22:43:30.965Z{"msg":"Crucible b3cec284-618e-4fb6-998c-7d358cb68d83 has session id: 68ad27c7-7c03-48c2-81ff-f276f9e37de3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496563152Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41062024-02-20T22:43:30.965Z{"msg":"Upstairs opts: Upstairs UUID: b3cec284-618e-4fb6-998c-7d358cb68d83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496588132Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41072024-02-20T22:43:30.965Z{"msg":"Crucible stats registered with UUID: b3cec284-618e-4fb6-998c-7d358cb68d83","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.49661802Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41082024-02-20T22:43:30.965Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.49663368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287}
41092024-02-20T22:43:30.983Z{"msg":"b3cec284-618e-4fb6-998c-7d358cb68d83 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496748486Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41102024-02-20T22:43:30.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496783245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41112024-02-20T22:43:30.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496811753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41122024-02-20T22:43:30.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496849382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41132024-02-20T22:43:30.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496878011Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41142024-02-20T22:43:30.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.49690195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41152024-02-20T22:43:30.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.496937759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41162024-02-20T22:43:30.983Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497015266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41172024-02-20T22:43:30.983Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497202439Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41182024-02-20T22:43:30.984Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497250058Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41192024-02-20T22:43:30.984Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497301686Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41202024-02-20T22:43:30.984Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497328795Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41212024-02-20T22:43:30.984Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497362184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41222024-02-20T22:43:30.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497729541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41232024-02-20T22:43:30.984Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497777819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41242024-02-20T22:43:30.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497808728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41252024-02-20T22:43:30.984Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497837777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41262024-02-20T22:43:30.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497868246Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41272024-02-20T22:43:30.984Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.497949383Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41282024-02-20T22:43:30.984Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.498413136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41292024-02-20T22:43:30.984Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.498451325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41302024-02-20T22:43:30.984Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.498514972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41312024-02-20T22:43:30.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499030914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41322024-02-20T22:43:30.984Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499087342Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41332024-02-20T22:43:30.984Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499122571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41342024-02-20T22:43:30.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.4991369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41352024-02-20T22:43:30.984Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499158309Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41362024-02-20T22:43:30.984Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499172499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41372024-02-20T22:43:30.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499193298Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41382024-02-20T22:43:30.984Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499207308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41392024-02-20T22:43:30.984Z{"msg":"[0]R flush_numbers[0..12]: [62, 64, 64, 66, 66, 66, 66, 64, 64, 66, 50, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499228207Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41402024-02-20T22:43:30.984Z{"msg":"[0]R generation[0..12]: [32, 35, 35, 37, 37, 37, 37, 35, 35, 37, 26, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499242506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41412024-02-20T22:43:30.984Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499263556Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41422024-02-20T22:43:30.985Z{"msg":"[1]R flush_numbers[0..12]: [62, 64, 64, 66, 66, 66, 66, 64, 64, 66, 50, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499277835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41432024-02-20T22:43:30.985Z{"msg":"[1]R generation[0..12]: [32, 35, 35, 37, 37, 37, 37, 35, 35, 37, 26, 37]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
41442024-02-20T22:43:30.985Z,"time":"2024-02-20T22:43:30.499298115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41452024-02-20T22:43:30.985Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499314974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41462024-02-20T22:43:30.985Z{"msg":"[2]R flush_numbers[0..12]: [62, 64, 64, 66, 66, 66, 66, 64, 64, 66, 50, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499335583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41472024-02-20T22:43:30.985Z{"msg":"[2]R generation[0..12]: [32, 35, 35, 37, 37, 37, 37, 35, 35, 37, 26, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499349573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41482024-02-20T22:43:30.985Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499364152Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41492024-02-20T22:43:30.985Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499385221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41502024-02-20T22:43:30.985Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499400141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41512024-02-20T22:43:30.985Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.49941973Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41522024-02-20T22:43:30.985Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499446109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41532024-02-20T22:43:30.985Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499469638Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41542024-02-20T22:43:30.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499499087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41552024-02-20T22:43:30.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499523316Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41562024-02-20T22:43:30.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499560245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41572024-02-20T22:43:30.985Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499583014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41582024-02-20T22:43:30.985Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499611893Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41592024-02-20T22:43:30.985Z{"msg":"b3cec284-618e-4fb6-998c-7d358cb68d83 is now active with session: 68ad27c7-7c03-48c2-81ff-f276f9e37de3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499633583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41602024-02-20T22:43:30.985Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.499674051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287,"session_id":"68ad27c7-7c03-48c2-81ff-f276f9e37de3"}
41612024-02-20T22:43:30.985ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41622024-02-20T22:43:30.985Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:30.49971102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287}
41632024-02-20T22:43:30.985ZLoading write count information from file "/tmp/test_repair_verify.data"
41642024-02-20T22:43:30.985ZRun Repair workload
41652024-02-20T22:43:30.985Z01/30 Write block 425 len 8192 data: 3 3
41662024-02-20T22:43:30.985Z02/30 Flush
41672024-02-20T22:43:30.986Z03/30 Write block 530 len 20480 data: 6 5 3 3 4
41682024-02-20T22:43:30.986Z04/30 Read block 529 len 8192
41692024-02-20T22:43:30.986Z05/30 Read block 136 len 4096
41702024-02-20T22:43:30.986Z06/30 Read block 136 len 24576
41712024-02-20T22:43:30.986Z07/30 Write block 508 len 16384 data: 4 4 5 5
41722024-02-20T22:43:30.995Z08/30 Write block 394 len 16384 data: 6 6 5 7
41732024-02-20T22:43:30.995Z09/30 Flush
41742024-02-20T22:43:30.995Z10/30 Flush
41752024-02-20T22:43:30.995Z11/30 Write block 446 len 28672 data: 4 4 5 5 5 4 6
41762024-02-20T22:43:30.995Z12/30 Read block 175 len 36864
41772024-02-20T22:43:30.995Z13/30 Write block 396 len 20480 data: 6 8 5 5 5
41782024-02-20T22:43:30.995Z14/30 Write block 588 len 40960 data: 6 6 6 5 4 3 4 4 4 4
41792024-02-20T22:43:30.995Z15/30 Flush
41802024-02-20T22:43:30.995Z16/30 Write block 281 len 36864 data: 4 4 5 5 4 5 4 3 3
41812024-02-20T22:43:30.995Z17/30 Read block 392 len 16384
41822024-02-20T22:43:30.996Z18/30 Flush
41832024-02-20T22:43:30.996Z19/30 Write block 455 len 16384 data: 6 6 6 6
41842024-02-20T22:43:30.996Z20/30 Flush
41852024-02-20T22:43:30.996Z21/30 Write block 227 len 32768 data: 3 4 5 6 5 6 6 6
41862024-02-20T22:43:30.996Z22/30 Read block 301 len 20480
41872024-02-20T22:43:30.996Z23/30 Flush
41882024-02-20T22:43:31.016Z24/30 Write block 356 len 32768 data: 5 3 4 6 7 7 6 8
41892024-02-20T22:43:31.016Z25/30 Write block 510 len 16384 data: 6 6 3 3
41902024-02-20T22:43:31.016Z26/30 Read block 273 len 32768
41912024-02-20T22:43:31.016Z27/30 Read block 419 len 24576
41922024-02-20T22:43:31.016Z28/30 Read block 97 len 12288
41932024-02-20T22:43:31.016Z29/30 Write block 39 len 4096 data: 5
41942024-02-20T22:43:31.016Z30/30 Write block 590 len 24576 data: 7 6 5 4 5 5
41952024-02-20T22:43:31.016Z----------------------------------------------------------------
41962024-02-20T22:43:31.016Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:28
41972024-02-20T22:43:31.016ZStates: Active Active Active
41982024-02-20T22:43:31.017ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41992024-02-20T22:43:31.017Z 3 Acked 1002 Write 5 Done Done Sent false
42002024-02-20T22:43:31.017Z 4 Acked 1003 Read 2 Done Done Sent false
42012024-02-20T22:43:31.017Z 5 Acked 1004 Read 1 Done Done Sent false
42022024-02-20T22:43:31.017Z 6 Acked 1005 Read 6 Done Done Sent false
42032024-02-20T22:43:31.017Z 7 Acked 1006 Write 4 Done Done Sent false
42042024-02-20T22:43:31.017Z 8 Acked 1007 Write 4 Done Done Sent false
42052024-02-20T22:43:31.017Z 9 Acked 1008 Flush 0 Done Done Sent false
42062024-02-20T22:43:31.017Z 10 Acked 1009 Flush 0 Done Done Sent false
42072024-02-20T22:43:31.017Z 11 Acked 1010 Write 7 Done Done Sent false
42082024-02-20T22:43:31.017Z 12 Acked 1011 Read 9 Done Done Sent false
42092024-02-20T22:43:31.017Z 13 Acked 1012 Write 5 Done Done Sent false
42102024-02-20T22:43:31.017Z 14 Acked 1013 Write 10 Done Done Sent false
42112024-02-20T22:43:31.017Z 15 Acked 1014 Flush 0 Done Done Sent false
42122024-02-20T22:43:31.017Z 16 Acked 1015 Write 9 Done Done Sent false
42132024-02-20T22:43:31.018Z 17 Acked 1016 Read 4 Done Done Sent false
42142024-02-20T22:43:31.018Z 18 Acked 1017 Flush 0 Done Done Sent false
42152024-02-20T22:43:31.018Z 19 Acked 1018 Write 4 Done Done Sent false
42162024-02-20T22:43:31.018Z 20 Acked 1019 Flush 0 Done Done Sent false
42172024-02-20T22:43:31.018Z 21 Acked 1020 Write 8 Done Done Sent false
42182024-02-20T22:43:31.018Z 22 Acked 1021 Read 5 Done Done Sent false
42192024-02-20T22:43:31.018Z 23 Acked 1022 Flush 0 Done Done Sent false
42202024-02-20T22:43:31.018Z 24 Acked 1023 Write 8 Done Done Sent false
42212024-02-20T22:43:31.018Z 25 Acked 1024 Write 4 Done Done Sent false
42222024-02-20T22:43:31.018Z 26 Acked 1025 Read 8 Done Done Sent false
42232024-02-20T22:43:31.019Z 27 Acked 1026 Read 6 Done Done Sent false
42242024-02-20T22:43:31.019Z 28 Acked 1027 Read 3 Done Done Sent false
42252024-02-20T22:43:31.019Z 29 Acked 1028 Write 1 Done Done Sent false
42262024-02-20T22:43:31.019Z 30 Acked 1029 Write 6 Sent Sent Sent false
42272024-02-20T22:43:31.020Z STATES DS:0 DS:1 DS:2 TOTAL
42282024-02-20T22:43:31.020Z New 0 0 0 0
42292024-02-20T22:43:31.020Z Sent 1 1 28 30
42302024-02-20T22:43:31.020Z Done 27 27 0 54
42312024-02-20T22:43:31.020Z Skipped 0 0 0 0
42322024-02-20T22:43:31.020Z Error 0 0 0 0
42332024-02-20T22:43:31.021ZLast Flush: 1022 1022 1001
42342024-02-20T22:43:31.021ZDownstairs last five completed: 1001 1000
42352024-02-20T22:43:31.021ZUpstairs last five completed: 30 29 28 27 26
42362024-02-20T22:43:31.021Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:30.532738398Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1287}
42372024-02-20T22:43:31.021ZWrote out file "/tmp/test_repair_verify.data"
42382024-02-20T22:43:31.021Z
42392024-02-20T22:43:31.033ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42402024-02-20T22:43:31.033ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42412024-02-20T22:43:31.033Z 0 000-019 32 32 32 62 62 62 F F F
42422024-02-20T22:43:31.033Z 1 020-039 35 35 35 64 64 64 T T F <---
42432024-02-20T22:43:31.033Z 2 040-059 35 35 35 64 64 64 F F F
42442024-02-20T22:43:31.033Z 3 060-079 37 37 37 66 66 66 F F F
42452024-02-20T22:43:31.033Z 4 080-099 37 37 37 66 66 66 F F F
42462024-02-20T22:43:31.033Z 5 100-119 37 37 37 66 66 66 F F F
42472024-02-20T22:43:31.033Z 6 120-139 37 37 37 66 66 66 F F F
42482024-02-20T22:43:31.033Z 7 140-159 35 35 35 64 64 64 F F F
42492024-02-20T22:43:31.033Z 8 160-179 35 35 35 64 64 64 F F F
42502024-02-20T22:43:31.033Z 9 180-199 37 37 37 66 66 66 F F F
42512024-02-20T22:43:31.033Z 10 200-219 26 26 26 50 50 50 F F F
42522024-02-20T22:43:31.033Z 11 220-239 38 38 37 73 73 66 F F F <---
42532024-02-20T22:43:31.034Z 12 240-259 36 36 36 65 65 65 F F F
42542024-02-20T22:43:31.034Z 13 260-279 37 37 37 66 66 66 F F F
42552024-02-20T22:43:31.034Z 14 280-299 38 38 37 71 71 66 F F F <---
42562024-02-20T22:43:31.034Z 15 300-319 37 37 37 66 66 66 F F F
42572024-02-20T22:43:31.034Z 16 320-339 37 37 37 66 66 66 F F F
42582024-02-20T22:43:31.034Z 17 340-359 32 32 32 62 62 62 T T F <---
42592024-02-20T22:43:31.034Z 18 360-379 37 37 37 66 66 66 T T F <---
42602024-02-20T22:43:31.034Z 19 380-399 38 38 37 70 70 66 F F F <---
42612024-02-20T22:43:31.034Z 20 400-419 38 38 37 70 70 66 F F F <---
42622024-02-20T22:43:31.034Z 21 420-439 38 38 38 67 67 67 F F F
42632024-02-20T22:43:31.034Z 22 440-459 38 38 37 72 72 66 F F F <---
42642024-02-20T22:43:31.034Z 23 460-479 36 36 36 65 65 65 F F F
42652024-02-20T22:43:31.034Z 24 480-499 35 35 35 64 64 64 F F F
42662024-02-20T22:43:31.034Z 25 500-519 38 38 35 68 68 64 T T F <---
42672024-02-20T22:43:31.034Z 26 520-539 38 38 35 68 68 64 F F F <---
42682024-02-20T22:43:31.034Z 27 540-559 36 36 36 65 65 65 F F F
42692024-02-20T22:43:31.034Z 28 560-579 37 37 37 66 66 66 F F F
42702024-02-20T22:43:31.034Z 29 580-599 38 38 25 70 70 49 T T F <---
42712024-02-20T22:43:31.034ZMax gen: 38, Max flush: 73
42722024-02-20T22:43:31.034ZError: Difference in extent metadata found!
42732024-02-20T22:43:31.034ZA Difference in extent metadata is expected here
42742024-02-20T22:43:31.034ZOn loop 19
42752024-02-20T22:43:31.034Z
42762024-02-20T22:43:31.034ZVerifying data now
42772024-02-20T22:43:35.580ZNo change after verify
42782024-02-20T22:43:35.580ZLoop: 19 Downstairs dump after verify (and repair):
42792024-02-20T22:43:35.595ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42802024-02-20T22:43:35.595Z 0 000-019 32 32 32 62 62 62 F F F
42812024-02-20T22:43:35.595Z 1 020-039 39 39 39 74 74 74 F F F
42822024-02-20T22:43:35.595Z 2 040-059 35 35 35 64 64 64 F F F
42832024-02-20T22:43:35.595Z 3 060-079 37 37 37 66 66 66 F F F
42842024-02-20T22:43:35.595Z 4 080-099 37 37 37 66 66 66 F F F
42852024-02-20T22:43:35.595Z 5 100-119 37 37 37 66 66 66 F F F
42862024-02-20T22:43:35.595Z 6 120-139 37 37 37 66 66 66 F F F
42872024-02-20T22:43:35.595Z 7 140-159 35 35 35 64 64 64 F F F
42882024-02-20T22:43:35.595Z 8 160-179 35 35 35 64 64 64 F F F
42892024-02-20T22:43:35.595Z 9 180-199 37 37 37 66 66 66 F F F
42902024-02-20T22:43:35.596Z 10 200-219 26 26 26 50 50 50 F F F
42912024-02-20T22:43:35.596Z 11 220-239 38 38 38 73 73 73 F F F
42922024-02-20T22:43:35.596Z 12 240-259 36 36 36 65 65 65 F F F
42932024-02-20T22:43:35.596Z 13 260-279 37 37 37 66 66 66 F F F
42942024-02-20T22:43:35.596Z 14 280-299 38 38 38 71 71 71 F F F
42952024-02-20T22:43:35.600Z 15 300-319 37 37 37 66 66 66 F F F
42962024-02-20T22:43:35.600Z 16 320-339 37 37 37 66 66 66 F F F
42972024-02-20T22:43:35.600Z 17 340-359 39 39 39 74 74 74 F F F
42982024-02-20T22:43:35.600Z 18 360-379 39 39 39 74 74 74 F F F
42992024-02-20T22:43:35.600Z 19 380-399 38 38 38 70 70 70 F F F
43002024-02-20T22:43:35.600Z 20 400-419 38 38 38 70 70 70 F F F
43012024-02-20T22:43:35.600Z 21 420-439 38 38 38 67 67 67 F F F
43022024-02-20T22:43:35.600Z 22 440-459 38 38 38 72 72 72 F F F
43032024-02-20T22:43:35.600Z 23 460-479 36 36 36 65 65 65 F F F
43042024-02-20T22:43:35.600Z 24 480-499 35 35 35 64 64 64 F F F
43052024-02-20T22:43:35.600Z 25 500-519 39 39 39 74 74 74 F F F
43062024-02-20T22:43:35.600Z 26 520-539 38 38 38 68 68 68 F F F
43072024-02-20T22:43:35.600Z 27 540-559 36 36 36 65 65 65 F F F
43082024-02-20T22:43:35.600Z 28 560-579 37 37 37 66 66 66 F F F
43092024-02-20T22:43:35.600Z 29 580-599 39 39 39 74 74 74 F F F
43102024-02-20T22:43:35.600ZMax gen: 39, Max flush: 74
43112024-02-20T22:43:35.600Z
43122024-02-20T22:43:35.601ZBegin loop 20
43132024-02-20T22:43:35.601ZDownstairs to restart: 1
43142024-02-20T22:43:35.617Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152272607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295}
43152024-02-20T22:43:35.628Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152649583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295}
43162024-02-20T22:43:35.628Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152699171Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295}
43172024-02-20T22:43:35.628Z{"msg":"Crucible 50e98478-4384-4278-bf7c-4cb4ef24727f has session id: 328313fe-6a91-482b-ba7b-991da074cd2a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15272859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43182024-02-20T22:43:35.628ZCrucible runtime is spawned
43192024-02-20T22:43:35.628Z{"msg":"Upstairs opts: Upstairs UUID: 50e98478-4384-4278-bf7c-4cb4ef24727f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152787768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43202024-02-20T22:43:35.628Z{"msg":"Crucible stats registered with UUID: 50e98478-4384-4278-bf7c-4cb4ef24727f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152828697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43212024-02-20T22:43:35.628Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152868905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295}
43222024-02-20T22:43:35.628Z{"msg":"50e98478-4384-4278-bf7c-4cb4ef24727f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.152966032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43232024-02-20T22:43:35.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15302158Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43242024-02-20T22:43:35.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153053389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43252024-02-20T22:43:35.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153079588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43262024-02-20T22:43:35.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153097397Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43272024-02-20T22:43:35.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153114337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43282024-02-20T22:43:35.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153138056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43292024-02-20T22:43:35.629Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153166665Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43302024-02-20T22:43:35.629Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153213633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43312024-02-20T22:43:35.629Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153262391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43322024-02-20T22:43:35.629Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15329142Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43332024-02-20T22:43:35.629Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153512082Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43342024-02-20T22:43:35.629Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.153564141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43352024-02-20T22:43:35.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154038854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43362024-02-20T22:43:35.629Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154089772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43372024-02-20T22:43:35.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15413796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43382024-02-20T22:43:35.629Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154170649Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43392024-02-20T22:43:35.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154199578Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43402024-02-20T22:43:35.629Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154226887Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43412024-02-20T22:43:35.629Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154670291Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43422024-02-20T22:43:35.629Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15470761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43432024-02-20T22:43:35.629Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.154732209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43442024-02-20T22:43:35.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155213752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43452024-02-20T22:43:35.629Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15526571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43462024-02-20T22:43:35.629Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155306558Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43472024-02-20T22:43:35.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155335007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43482024-02-20T22:43:35.629Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155349917Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43492024-02-20T22:43:35.629Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155371206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43502024-02-20T22:43:35.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155384295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43512024-02-20T22:43:35.629Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155404365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43522024-02-20T22:43:35.629Z{"msg":"[0]R flush_numbers[0..12]: [62, 74, 64, 66, 66, 66, 66, 64, 64, 66, 50, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155418564Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43532024-02-20T22:43:35.629Z{"msg":"[0]R generation[0..12]: [32, 39, 35, 37, 37, 37, 37, 35, 35, 37, 26, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155439624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43542024-02-20T22:43:35.630Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155454273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43552024-02-20T22:43:35.630Z{"msg":"[1]R flush_numbers[0..12]: [62, 74, 64, 66, 66, 66, 66, 64, 64, 66, 50, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155480332Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43562024-02-20T22:43:35.630Z{"msg":"[1]R generation[0..12]: [32, 39, 35, 37, 37, 37, 37, 35, 35, 37, 26, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155500711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43572024-02-20T22:43:35.630Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
43582024-02-20T22:43:35.630Z,"time":"2024-02-20T22:43:35.155517561Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43592024-02-20T22:43:35.630Z{"msg":"[2]R flush_numbers[0..12]: [62, 74, 64, 66, 66, 66, 66, 64, 64, 66, 50, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.15554487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43602024-02-20T22:43:35.630Z{"msg":"[2]R generation[0..12]: [32, 39, 35, 37, 37, 37, 37, 35, 35, 37, 26, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155559499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43612024-02-20T22:43:35.630Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155581219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43622024-02-20T22:43:35.630Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155594858Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43632024-02-20T22:43:35.630Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155615437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43642024-02-20T22:43:35.630Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155628837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43652024-02-20T22:43:35.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155642226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43662024-02-20T22:43:35.630Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155662305Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43672024-02-20T22:43:35.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155675145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43682024-02-20T22:43:35.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155695644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43692024-02-20T22:43:35.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155709184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43702024-02-20T22:43:35.630Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155729343Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43712024-02-20T22:43:35.630Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155741833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43722024-02-20T22:43:35.630Z{"msg":"50e98478-4384-4278-bf7c-4cb4ef24727f is now active with session: 328313fe-6a91-482b-ba7b-991da074cd2a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155761512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43732024-02-20T22:43:35.630Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155773912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295,"session_id":"328313fe-6a91-482b-ba7b-991da074cd2a"}
43742024-02-20T22:43:35.630Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:35.155793041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295}
43752024-02-20T22:43:35.630ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43762024-02-20T22:43:35.630ZLoading write count information from file "/tmp/test_repair_verify.data"
43772024-02-20T22:43:35.630ZRun Repair workload
43782024-02-20T22:43:35.630Z01/30 Write block 485 len 24576 data: 4 4 4 4 2 2
43792024-02-20T22:43:35.631Z02/30 Write block 314 len 28672 data: 4 6 5 5 4 5 6
43802024-02-20T22:43:35.637Z03/30 Read block 525 len 28672
43812024-02-20T22:43:35.637Z04/30 Read block 16 len 16384
43822024-02-20T22:43:35.637Z05/30 Read block 380 len 28672
43832024-02-20T22:43:35.637Z06/30 Write block 33 len 4096 data: 7
43842024-02-20T22:43:35.637Z07/30 Write block 564 len 20480 data: 3 2 3 3 3
43852024-02-20T22:43:35.637Z08/30 Read block 584 len 12288
43862024-02-20T22:43:35.637Z09/30 Write block 29 len 36864 data: 9 9 8 8 8 6 6 5 5
43872024-02-20T22:43:35.638Z10/30 Write block 253 len 4096 data: 3
43882024-02-20T22:43:35.638Z11/30 Read block 492 len 40960
43892024-02-20T22:43:35.638Z12/30 Flush
43902024-02-20T22:43:35.638Z13/30 Write block 29 len 4096 data: 10
43912024-02-20T22:43:35.638Z14/30 Write block 247 len 4096 data: 3
43922024-02-20T22:43:35.638Z15/30 Read block 297 len 40960
43932024-02-20T22:43:35.638Z16/30 Read block 358 len 4096
43942024-02-20T22:43:35.638Z17/30 Write block 457 len 36864 data: 7 7 5 5 5 8 8 7 9
43952024-02-20T22:43:35.645Z18/30 Write block 392 len 8192 data: 6 6
43962024-02-20T22:43:35.645Z19/30 Write block 439 len 36864 data: 4 4 4 4 5 5 5 5 5
43972024-02-20T22:43:35.645Z20/30 Read block 372 len 16384
43982024-02-20T22:43:35.645Z21/30 Read block 414 len 28672
43992024-02-20T22:43:35.645Z22/30 Write block 266 len 8192 data: 5 5
44002024-02-20T22:43:35.645Z23/30 Read block 310 len 28672
44012024-02-20T22:43:35.652Z24/30 Write block 477 len 32768 data: 3 3 4 6 5 5 6 6
44022024-02-20T22:43:35.652Z25/30 Write block 38 len 8192 data: 5 6
44032024-02-20T22:43:35.652Z26/30 Write block 445 len 8192 data: 6 6
44042024-02-20T22:43:35.652Z27/30 Write block 593 len 8192 data: 5 6
44052024-02-20T22:43:35.652Z28/30 Read block 364 len 16384
44062024-02-20T22:43:35.652Z29/30 Read block 227 len 36864
44072024-02-20T22:43:35.652Z30/30 Write block 218 len 12288 data: 2 3 3
44082024-02-20T22:43:35.652Z----------------------------------------------------------------
44092024-02-20T22:43:35.652Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44102024-02-20T22:43:35.652ZStates: Active Active Active
44112024-02-20T22:43:35.652ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44122024-02-20T22:43:35.652Z 1 Acked 1000 Write 6 Done Done Done false
44132024-02-20T22:43:35.652Z 2 Acked 1001 Write 7 Done Done Done false
44142024-02-20T22:43:35.652Z 3 Acked 1002 Read 7 Done Sent Done false
44152024-02-20T22:43:35.652Z 4 Acked 1003 Read 4 Done Sent Done false
44162024-02-20T22:43:35.652Z 5 Acked 1004 Read 7 Done Sent Done false
44172024-02-20T22:43:35.652Z 6 Acked 1005 Write 1 Done Sent Done false
44182024-02-20T22:43:35.653Z 7 Acked 1006 Write 5 Done Sent Done false
44192024-02-20T22:43:35.653Z 8 Acked 1007 Read 3 Done Sent Done false
44202024-02-20T22:43:35.653Z 9 Acked 1008 Write 9 Done Sent Done false
44212024-02-20T22:43:35.653Z 10 Acked 1009 Write 1 Done Sent Done false
44222024-02-20T22:43:35.653Z 11 Acked 1010 Read 10 Done Sent Done false
44232024-02-20T22:43:35.653Z 12 Acked 1011 Flush 0 Done Sent Done false
44242024-02-20T22:43:35.653Z 13 Acked 1012 Write 1 Done Sent Done false
44252024-02-20T22:43:35.653Z 14 Acked 1013 Write 1 Done Sent Done false
44262024-02-20T22:43:35.653Z 15 Acked 1014 Read 10 Done Sent Done false
44272024-02-20T22:43:35.653Z 16 Acked 1015 Read 1 Done Sent Done false
44282024-02-20T22:43:35.653Z 17 Acked 1016 Write 9 Done Sent Done false
44292024-02-20T22:43:35.653Z 18 Acked 1017 Write 2 Done Sent Done false
44302024-02-20T22:43:35.653Z 19 Acked 1018 Write 9 Done Sent Done false
44312024-02-20T22:43:35.653Z 20 Acked 1019 Read 4 Done Sent Done false
44322024-02-20T22:43:35.653Z 21 Acked 1020 Read 7 Done Sent Done false
44332024-02-20T22:43:35.653Z 22 Acked 1021 Write 2 Done Sent Done false
44342024-02-20T22:43:35.653Z 23 Acked 1022 Read 7 Done Sent Done false
44352024-02-20T22:43:35.653Z 24 Acked 1023 Write 8 Done Sent Done false
44362024-02-20T22:43:35.653Z 25 Acked 1024 Write 2 Done Sent Done false
44372024-02-20T22:43:35.653Z 26 Acked 1025 Write 2 Done Sent Done false
44382024-02-20T22:43:35.653Z 27 Acked 1026 Write 2 Done Sent Done false
44392024-02-20T22:43:35.653Z 28 Acked 1027 Read 4 Done Sent Done false
44402024-02-20T22:43:35.653Z 29 Acked 1028 Read 9 Done Sent Done false
44412024-02-20T22:43:35.653Z 30 Acked 1029 Write 3 Sent Sent Sent false
44422024-02-20T22:43:35.653Z STATES DS:0 DS:1 DS:2 TOTAL
44432024-02-20T22:43:35.654Z New 0 0 0 0
44442024-02-20T22:43:35.654Z Sent 1 28 1 30
44452024-02-20T22:43:35.654Z Done 29 2 29 60
44462024-02-20T22:43:35.654Z Skipped 0 0 0 0
44472024-02-20T22:43:35.654Z Error 0 0 0 0
44482024-02-20T22:43:35.654ZLast Flush: 1011 0 1011
44492024-02-20T22:43:35.654ZDownstairs last five completed:
44502024-02-20T22:43:35.654ZUpstairs last five completed: 30 29 28 27 26
44512024-02-20T22:43:35.654Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:35.182869232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1295}
44522024-02-20T22:43:35.654ZWrote out file "/tmp/test_repair_verify.data"
44532024-02-20T22:43:35.658Z
44542024-02-20T22:43:35.658ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44552024-02-20T22:43:35.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44562024-02-20T22:43:35.673Z 0 000-019 32 32 32 62 62 62 F F F
44572024-02-20T22:43:35.673Z 1 020-039 40 39 40 75 74 75 T F T <---
44582024-02-20T22:43:35.673Z 2 040-059 35 35 35 64 64 64 F F F
44592024-02-20T22:43:35.673Z 3 060-079 37 37 37 66 66 66 F F F
44602024-02-20T22:43:35.674Z 4 080-099 37 37 37 66 66 66 F F F
44612024-02-20T22:43:35.674Z 5 100-119 37 37 37 66 66 66 F F F
44622024-02-20T22:43:35.674Z 6 120-139 37 37 37 66 66 66 F F F
44632024-02-20T22:43:35.674Z 7 140-159 35 35 35 64 64 64 F F F
44642024-02-20T22:43:35.674Z 8 160-179 35 35 35 64 64 64 F F F
44652024-02-20T22:43:35.674Z 9 180-199 37 37 37 66 66 66 F F F
44662024-02-20T22:43:35.678Z 10 200-219 26 26 26 50 50 50 T F T <---
44672024-02-20T22:43:35.678Z 11 220-239 38 38 38 73 73 73 T F T <---
44682024-02-20T22:43:35.678Z 12 240-259 40 36 40 75 65 75 T F T <---
44692024-02-20T22:43:35.678Z 13 260-279 37 37 37 66 66 66 T F T <---
44702024-02-20T22:43:35.678Z 14 280-299 38 38 38 71 71 71 F F F
44712024-02-20T22:43:35.678Z 15 300-319 40 37 40 75 66 75 F T F <---
44722024-02-20T22:43:35.678Z 16 320-339 40 37 40 75 66 75 F T F <---
44732024-02-20T22:43:35.678Z 17 340-359 39 39 39 74 74 74 F F F
44742024-02-20T22:43:35.678Z 18 360-379 39 39 39 74 74 74 F F F
44752024-02-20T22:43:35.679Z 19 380-399 38 38 38 70 70 70 T F T <---
44762024-02-20T22:43:35.679Z 20 400-419 38 38 38 70 70 70 F F F
44772024-02-20T22:43:35.679Z 21 420-439 38 38 38 67 67 67 T F T <---
44782024-02-20T22:43:35.679Z 22 440-459 38 38 38 72 72 72 T F T <---
44792024-02-20T22:43:35.679Z 23 460-479 36 36 36 65 65 65 T F T <---
44802024-02-20T22:43:35.679Z 24 480-499 40 35 40 75 64 75 T T T <---
44812024-02-20T22:43:35.679Z 25 500-519 39 39 39 74 74 74 F F F
44822024-02-20T22:43:35.679Z 26 520-539 38 38 38 68 68 68 F F F
44832024-02-20T22:43:35.679Z 27 540-559 36 36 36 65 65 65 F F F
44842024-02-20T22:43:35.679Z 28 560-579 40 37 40 75 66 75 F F F <---
44852024-02-20T22:43:35.679Z 29 580-599 39 39 39 74 74 74 T F T <---
44862024-02-20T22:43:35.679ZMax gen: 40, Max flush: 75
44872024-02-20T22:43:35.679ZError: Difference in extent metadata found!
44882024-02-20T22:43:35.679ZA Difference in extent metadata is expected here
44892024-02-20T22:43:35.679ZOn loop 20
44902024-02-20T22:43:35.679Z
44912024-02-20T22:43:35.683ZVerifying data now
44922024-02-20T22:43:41.385ZNo change after verify
44932024-02-20T22:43:41.385ZLoop: 20 Downstairs dump after verify (and repair):
44942024-02-20T22:43:41.409ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44952024-02-20T22:43:41.409Z 0 000-019 32 32 32 62 62 62 F F F
44962024-02-20T22:43:41.409Z 1 020-039 41 41 41 76 76 76 F F F
44972024-02-20T22:43:41.409Z 2 040-059 35 35 35 64 64 64 F F F
44982024-02-20T22:43:41.409Z 3 060-079 37 37 37 66 66 66 F F F
44992024-02-20T22:43:41.410Z 4 080-099 37 37 37 66 66 66 F F F
45002024-02-20T22:43:41.410Z 5 100-119 37 37 37 66 66 66 F F F
45012024-02-20T22:43:41.410Z 6 120-139 37 37 37 66 66 66 F F F
45022024-02-20T22:43:41.410Z 7 140-159 35 35 35 64 64 64 F F F
45032024-02-20T22:43:41.410Z 8 160-179 35 35 35 64 64 64 F F F
45042024-02-20T22:43:41.410Z 9 180-199 37 37 37 66 66 66 F F F
45052024-02-20T22:43:41.410Z 10 200-219 41 41 41 76 76 76 F F F
45062024-02-20T22:43:41.410Z 11 220-239 41 41 41 76 76 76 F F F
45072024-02-20T22:43:41.410Z 12 240-259 41 41 41 76 76 76 F F F
45082024-02-20T22:43:41.410Z 13 260-279 41 41 41 76 76 76 F F F
45092024-02-20T22:43:41.410Z 14 280-299 38 38 38 71 71 71 F F F
45102024-02-20T22:43:41.410Z 15 300-319 40 40 40 75 75 75 F F F
45112024-02-20T22:43:41.410Z 16 320-339 40 40 40 75 75 75 F F F
45122024-02-20T22:43:41.410Z 17 340-359 39 39 39 74 74 74 F F F
45132024-02-20T22:43:41.411Z 18 360-379 39 39 39 74 74 74 F F F
45142024-02-20T22:43:41.411Z 19 380-399 41 41 41 76 76 76 F F F
45152024-02-20T22:43:41.411Z 20 400-419 38 38 38 70 70 70 F F F
45162024-02-20T22:43:41.411Z 21 420-439 41 41 41 76 76 76 F F F
45172024-02-20T22:43:41.411Z 22 440-459 41 41 41 76 76 76 F F F
45182024-02-20T22:43:41.411Z 23 460-479 41 41 41 76 76 76 F F F
45192024-02-20T22:43:41.411Z 24 480-499 41 41 41 76 76 76 F F F
45202024-02-20T22:43:41.411Z 25 500-519 39 39 39 74 74 74 F F F
45212024-02-20T22:43:41.411Z 26 520-539 38 38 38 68 68 68 F F F
45222024-02-20T22:43:41.411Z 27 540-559 36 36 36 65 65 65 F F F
45232024-02-20T22:43:41.411Z 28 560-579 40 40 40 75 75 75 F F F
45242024-02-20T22:43:41.411Z 29 580-599 41 41 41 76 76 76 F F F
45252024-02-20T22:43:41.411ZMax gen: 41, Max flush: 76
45262024-02-20T22:43:41.411Z
45272024-02-20T22:43:41.412ZBegin loop 21
45282024-02-20T22:43:41.412ZDownstairs to restart: 0
45292024-02-20T22:43:41.440Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947173899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303}
45302024-02-20T22:43:41.440Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947489997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303}
45312024-02-20T22:43:41.440Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
45322024-02-20T22:43:41.440Ztime":"2024-02-20T22:43:40.947535466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303}
45332024-02-20T22:43:41.440Z{"msg":"Crucible b94b10e6-0ec6-4a1d-9b74-6cd4185ed7fc has session id: 9481b91b-4b83-45d0-b111-66e6d295eac8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947569194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45342024-02-20T22:43:41.440Z{"msg":"Upstairs opts: Upstairs UUID: b94b10e6-0ec6-4a1d-9b74-6cd4185ed7fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947609453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45352024-02-20T22:43:41.441Z{"msg":"Crucible stats registered with UUID: b94b10e6-0ec6-4a1d-9b74-6cd4185ed7fc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947625552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45362024-02-20T22:43:41.441Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947655911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303}
45372024-02-20T22:43:41.441Z{"msg":"b94b10e6-0ec6-4a1d-9b74-6cd4185ed7fc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947680221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45382024-02-20T22:43:41.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.94770583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45392024-02-20T22:43:41.441Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947729869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45402024-02-20T22:43:41.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947771877Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45412024-02-20T22:43:41.441Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947796897Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45422024-02-20T22:43:41.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947835145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45432024-02-20T22:43:41.441Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947866744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45442024-02-20T22:43:41.441Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.947913342Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45452024-02-20T22:43:41.441Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.94797632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45462024-02-20T22:43:41.442Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948019018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45472024-02-20T22:43:41.442Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948078906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45482024-02-20T22:43:41.442Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948113555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45492024-02-20T22:43:41.442Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948160904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45502024-02-20T22:43:41.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948917966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45512024-02-20T22:43:41.442Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948967444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45522024-02-20T22:43:41.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.948994674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45532024-02-20T22:43:41.442Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949036412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45542024-02-20T22:43:41.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.9490827Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45552024-02-20T22:43:41.442Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949121809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45562024-02-20T22:43:41.442Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949496896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45572024-02-20T22:43:41.442Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949554254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45582024-02-20T22:43:41.442Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949755136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45592024-02-20T22:43:41.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949955209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45602024-02-20T22:43:41.443Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.949997518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45612024-02-20T22:43:41.443Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950015107Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45622024-02-20T22:43:41.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950044276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45632024-02-20T22:43:41.443Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950069435Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45642024-02-20T22:43:41.443Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950099614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45652024-02-20T22:43:41.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.95021146Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45662024-02-20T22:43:41.443Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950256338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45672024-02-20T22:43:41.443Z{"msg":"[0]R flush_numbers[0..12]: [62, 76, 64, 66, 66, 66, 66, 64, 64, 66, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950274548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45682024-02-20T22:43:41.444Z{"msg":"[0]R generation[0..12]: [32, 41, 35, 37, 37, 37, 37, 35, 35, 37, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950308136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45692024-02-20T22:43:41.444Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950340366Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45702024-02-20T22:43:41.444Z{"msg":"[1]R flush_numbers[0..12]: [62, 76, 64, 66, 66, 66, 66, 64, 64, 66, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950362575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45712024-02-20T22:43:41.444Z{"msg":"[1]R generation[0..12]: [32, 41, 35, 37, 37, 37, 37, 35, 35, 37, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950397803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
45722024-02-20T22:43:41.444Z9481b91b-4b83-45d0-b111-66e6d295eac8"}
45732024-02-20T22:43:41.444Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950426822Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45742024-02-20T22:43:41.444Z{"msg":"[2]R flush_numbers[0..12]: [62, 76, 64, 66, 66, 66, 66, 64, 64, 66, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950455841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45752024-02-20T22:43:41.444Z{"msg":"[2]R generation[0..12]: [32, 41, 35, 37, 37, 37, 37, 35, 35, 37, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950474581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45762024-02-20T22:43:41.444Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950504489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45772024-02-20T22:43:41.445Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950533019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45782024-02-20T22:43:41.445Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950567037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45792024-02-20T22:43:41.445Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950598336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45802024-02-20T22:43:41.445Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950629685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45812024-02-20T22:43:41.445Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950662164Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45822024-02-20T22:43:41.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950691413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45832024-02-20T22:43:41.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950724052Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45842024-02-20T22:43:41.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950755051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45852024-02-20T22:43:41.445Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950787719Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45862024-02-20T22:43:41.445Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45872024-02-20T22:43:41.445Z2024-02-20T22:43:40.950817928Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45882024-02-20T22:43:41.445Z{"msg":"b94b10e6-0ec6-4a1d-9b74-6cd4185ed7fc is now active with session: 9481b91b-4b83-45d0-b111-66e6d295eac8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950846427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45892024-02-20T22:43:41.445Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950873626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303,"session_id":"9481b91b-4b83-45d0-b111-66e6d295eac8"}
45902024-02-20T22:43:41.445Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:40.950890586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303}
45912024-02-20T22:43:41.445ZLoading write count information from file "/tmp/test_repair_verify.data"
45922024-02-20T22:43:41.445ZRun Repair workload
45932024-02-20T22:43:41.445Z01/30 Write block 27 len 36864 data: 8 9 11 10 9 9 9 7 7
45942024-02-20T22:43:41.445Z02/30 Write block 362 len 20480 data: 7 9 8 11 9
45952024-02-20T22:43:41.445Z03/30 Write block 380 len 4096 data: 3
45962024-02-20T22:43:41.453Z04/30 Write block 406 len 40960 data: 6 6 5 8 8 8 8 7 7 7
45972024-02-20T22:43:41.453Z05/30 Write block 548 len 12288 data: 4 5 5
45982024-02-20T22:43:41.453Z06/30 Flush
45992024-02-20T22:43:41.453Z07/30 Flush
46002024-02-20T22:43:41.453Z08/30 Write block 376 len 8192 data: 5 6
46012024-02-20T22:43:41.454Z09/30 Write block 170 len 16384 data: 5 5 5 5
46022024-02-20T22:43:41.454Z10/30 Read block 268 len 28672
46032024-02-20T22:43:41.454Z11/30 Write block 27 len 28672 data: 9 10 12 11 10 10 10
46042024-02-20T22:43:41.454Z12/30 Flush
46052024-02-20T22:43:41.454Z13/30 Write block 116 len 8192 data: 6 7
46062024-02-20T22:43:41.454Z14/30 Read block 186 len 20480
46072024-02-20T22:43:41.454Z15/30 Write block 442 len 8192 data: 5 6
46082024-02-20T22:43:41.454Z16/30 Write block 315 len 36864 data: 7 6 6 5 6 7 6 7 4
46092024-02-20T22:43:41.454Z17/30 Flush
46102024-02-20T22:43:41.454Z18/30 Write block 100 len 4096 data: 5
46112024-02-20T22:43:41.454Z19/30 Read block 333 len 40960
46122024-02-20T22:43:41.454Z20/30 Read block 350 len 8192
46132024-02-20T22:43:41.454Z21/30 Read block 474 len 4096
46142024-02-20T22:43:41.454Z22/30 Write block 492 len 28672 data: 3 5 5 5 6 7 6
46152024-02-20T22:43:41.455Z23/30 Write block 456 len 16384 data: 7 8 8 6
46162024-02-20T22:43:41.455Z24/30 Write block 16 len 8192 data: 5 7
46172024-02-20T22:43:41.455Z25/30 Read block 100 len 20480
46182024-02-20T22:43:41.455Z26/30 Write block 230 len 32768 data: 7 6 7 7 7 5 4 4
46192024-02-20T22:43:41.455Z27/30 Write block 270 len 36864 data: 4 4 4 3 3 3 2 3 3
46202024-02-20T22:43:41.455Z28/30 Read block 543 len 8192
46212024-02-20T22:43:41.455Z29/30 Write block 333 len 36864 data: 7 6 5 5 5 5 6 5 4
46222024-02-20T22:43:41.455Z30/30 Write block 595 len 16384 data: 6 5 5 3
46232024-02-20T22:43:41.455Z----------------------------------------------------------------
46242024-02-20T22:43:41.455Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46252024-02-20T22:43:41.455ZStates: Active Active Active
46262024-02-20T22:43:41.455ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46272024-02-20T22:43:41.455Z 1 Acked 1000 Write 9 Done Done Done false
46282024-02-20T22:43:41.456Z 2 Acked 1001 Write 5 Sent Done Done false
46292024-02-20T22:43:41.456Z 3 Acked 1002 Write 1 Sent Done Done false
46302024-02-20T22:43:41.456Z 4 Acked 1003 Write 10 Sent Done Done false
46312024-02-20T22:43:41.456Z 5 Acked 1004 Write 3 Sent Done Done false
46322024-02-20T22:43:41.456Z 6 Acked 1005 Flush 0 Sent Done Done false
46332024-02-20T22:43:41.456Z 7 Acked 1006 Flush 0 Sent Done Done false
46342024-02-20T22:43:41.456Z 8 Acked 1007 Write 2 Sent Done Done false
46352024-02-20T22:43:41.456Z 9 Acked 1008 Write 4 Sent Done Done false
46362024-02-20T22:43:41.456Z 10 Acked 1009 Read 7 Sent Done Done false
46372024-02-20T22:43:41.456Z 11 Acked 1010 Write 7 Sent Done Done false
46382024-02-20T22:43:41.456Z 12 Acked 1011 Flush 0 Sent Done Done false
46392024-02-20T22:43:41.456Z 13 Acked 1012 Write 2 Sent Done Done false
46402024-02-20T22:43:41.456Z 14 Acked 1013 Read 5 Sent Done Done false
46412024-02-20T22:43:41.456Z 15 Acked 1014 Write 2 Sent Done Done false
46422024-02-20T22:43:41.456Z 16 Acked 1015 Write 9 Sent Done Done false
46432024-02-20T22:43:41.456Z 17 Acked 1016 Flush 0 Sent Done Done false
46442024-02-20T22:43:41.456Z 18 Acked 1017 Write 1 Sent Done Done false
46452024-02-20T22:43:41.457Z 19 Acked 1018 Read 10 Sent Done Done false
46462024-02-20T22:43:41.457Z 20 Acked 1019 Read 2 Sent Done Done false
46472024-02-20T22:43:41.457Z 21 Acked 1020 Read 1 Sent Done Done false
46482024-02-20T22:43:41.457Z 22 Acked 1021 Write 7 Sent Done Done false
46492024-02-20T22:43:41.457Z 23 Acked 1022 Write 4 Sent Done Done false
46502024-02-20T22:43:41.457Z 24 Acked 1023 Write 2 Sent Done Done false
46512024-02-20T22:43:41.457Z 25 Acked 1024 Read 5 Sent Done Done false
46522024-02-20T22:43:41.457Z 26 Acked 1025 Write 8 Sent Done Done false
46532024-02-20T22:43:41.457Z 27 Acked 1026 Write 9 Sent Done Done false
46542024-02-20T22:43:41.457Z 28 Acked 1027 Read 2 Sent Done Done false
46552024-02-20T22:43:41.457Z 29 Acked 1028 Write 9 Sent Sent Sent false
46562024-02-20T22:43:41.457Z 30 Acked 1029 Write 4 Sent Sent Sent false
46572024-02-20T22:43:41.457Z STATES DS:0 DS:1 DS:2 TOTAL
46582024-02-20T22:43:41.457Z New 0 0 0 0
46592024-02-20T22:43:41.457Z Sent 29 2 2 33
46602024-02-20T22:43:41.457Z Done 1 28 28 57
46612024-02-20T22:43:41.457Z Skipped 0 0 0 0
46622024-02-20T22:43:41.458Z Error 0 0 0 0
46632024-02-20T22:43:41.458ZLast Flush: 0 1016 1016
46642024-02-20T22:43:41.458ZDownstairs last five completed:
46652024-02-20T22:43:41.458ZUpstairs last five completed: 30 29 28 27 26
46662024-02-20T22:43:41.458Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:40.980566764Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1303}
46672024-02-20T22:43:41.458ZWrote out file "/tmp/test_repair_verify.data"
46682024-02-20T22:43:41.458Z
46692024-02-20T22:43:41.464ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46702024-02-20T22:43:41.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46712024-02-20T22:43:41.472Z 0 000-019 32 32 32 62 62 62 F T T <---
46722024-02-20T22:43:41.472Z 1 020-039 41 42 42 76 79 79 T F F <---
46732024-02-20T22:43:41.472Z 2 040-059 35 35 35 64 64 64 F F F
46742024-02-20T22:43:41.472Z 3 060-079 37 37 37 66 66 66 F F F
46752024-02-20T22:43:41.472Z 4 080-099 37 37 37 66 66 66 F F F
46762024-02-20T22:43:41.472Z 5 100-119 37 42 42 66 80 80 F T T <---
46772024-02-20T22:43:41.473Z 6 120-139 37 37 37 66 66 66 F F F
46782024-02-20T22:43:41.473Z 7 140-159 35 35 35 64 64 64 F F F
46792024-02-20T22:43:41.473Z 8 160-179 35 42 42 64 79 79 F F F <---
46802024-02-20T22:43:41.478Z 9 180-199 37 37 37 66 66 66 F F F
46812024-02-20T22:43:41.479Z 10 200-219 41 41 41 76 76 76 F F F
46822024-02-20T22:43:41.479Z 11 220-239 41 41 41 76 76 76 F T T <---
46832024-02-20T22:43:41.479Z 12 240-259 41 41 41 76 76 76 F F F
46842024-02-20T22:43:41.479Z 13 260-279 41 41 41 76 76 76 F T T <---
46852024-02-20T22:43:41.479Z 14 280-299 38 38 38 71 71 71 F F F
46862024-02-20T22:43:41.479Z 15 300-319 40 42 42 75 80 80 F F F <---
46872024-02-20T22:43:41.479Z 16 320-339 40 42 42 75 80 80 F T T <---
46882024-02-20T22:43:41.479Z 17 340-359 39 39 39 74 74 74 F T T <---
46892024-02-20T22:43:41.479Z 18 360-379 39 42 42 74 79 79 F F F <---
46902024-02-20T22:43:41.479Z 19 380-399 41 42 42 76 77 77 F F F <---
46912024-02-20T22:43:41.479Z 20 400-419 38 42 42 70 77 77 F F F <---
46922024-02-20T22:43:41.480Z 21 420-439 41 41 41 76 76 76 F F F
46932024-02-20T22:43:41.480Z 22 440-459 41 42 42 76 80 80 F T T <---
46942024-02-20T22:43:41.480Z 23 460-479 41 41 41 76 76 76 F F F
46952024-02-20T22:43:41.480Z 24 480-499 41 41 41 76 76 76 F T T <---
46962024-02-20T22:43:41.480Z 25 500-519 39 39 39 74 74 74 F F F
46972024-02-20T22:43:41.480Z 26 520-539 38 38 38 68 68 68 F F F
46982024-02-20T22:43:41.480Z 27 540-559 36 42 42 65 77 77 F F F <---
46992024-02-20T22:43:41.480Z 28 560-579 40 40 40 75 75 75 F F F
47002024-02-20T22:43:41.480Z 29 580-599 41 41 41 76 76 76 F T T <---
47012024-02-20T22:43:41.480ZMax gen: 42, Max flush: 80
47022024-02-20T22:43:41.480ZError: Difference in extent metadata found!
47032024-02-20T22:43:41.480ZA Difference in extent metadata is expected here
47042024-02-20T22:43:41.481ZOn loop 21
47052024-02-20T22:43:41.481Z
47062024-02-20T22:43:41.486ZVerifying data now
47072024-02-20T22:43:46.233ZNo change after verify
47082024-02-20T22:43:46.233ZLoop: 21 Downstairs dump after verify (and repair):
47092024-02-20T22:43:46.247ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47102024-02-20T22:43:46.247Z 0 000-019 43 43 43 81 81 81 F F F
47112024-02-20T22:43:46.248Z 1 020-039 42 42 42 79 79 79 F F F
47122024-02-20T22:43:46.248Z 2 040-059 35 35 35 64 64 64 F F F
47132024-02-20T22:43:46.248Z 3 060-079 37 37 37 66 66 66 F F F
47142024-02-20T22:43:46.248Z 4 080-099 37 37 37 66 66 66 F F F
47152024-02-20T22:43:46.248Z 5 100-119 43 43 43 81 81 81 F F F
47162024-02-20T22:43:46.248Z 6 120-139 37 37 37 66 66 66 F F F
47172024-02-20T22:43:46.248Z 7 140-159 35 35 35 64 64 64 F F F
47182024-02-20T22:43:46.248Z 8 160-179 42 42 42 79 79 79 F F F
47192024-02-20T22:43:46.248Z 9 180-199 37 37 37 66 66 66 F F F
47202024-02-20T22:43:46.248Z 10 200-219 41 41 41 76 76 76 F F F
47212024-02-20T22:43:46.248Z 11 220-239 43 43 43 81 81 81 F F F
47222024-02-20T22:43:46.255Z 12 240-259 41 41 41 76 76 76 F F F
47232024-02-20T22:43:46.255Z 13 260-279 43 43 43 81 81 81 F F F
47242024-02-20T22:43:46.255Z 14 280-299 38 38 38 71 71 71 F F F
47252024-02-20T22:43:46.255Z 15 300-319 42 42 42 80 80 80 F F F
47262024-02-20T22:43:46.255Z 16 320-339 43 43 43 81 81 81 F F F
47272024-02-20T22:43:46.255Z 17 340-359 43 43 43 81 81 81 F F F
47282024-02-20T22:43:46.255Z 18 360-379 42 42 42 79 79 79 F F F
47292024-02-20T22:43:46.255Z 19 380-399 42 42 42 77 77 77 F F F
47302024-02-20T22:43:46.255Z 20 400-419 42 42 42 77 77 77 F F F
47312024-02-20T22:43:46.255Z 21 420-439 41 41 41 76 76 76 F F F
47322024-02-20T22:43:46.255Z 22 440-459 43 43 43 81 81 81 F F F
47332024-02-20T22:43:46.255Z 23 460-479 41 41 41 76 76 76 F F F
47342024-02-20T22:43:46.256Z 24 480-499 43 43 43 81 81 81 F F F
47352024-02-20T22:43:46.256Z 25 500-519 39 39 39 74 74 74 F F F
47362024-02-20T22:43:46.256Z 26 520-539 38 38 38 68 68 68 F F F
47372024-02-20T22:43:46.256Z 27 540-559 42 42 42 77 77 77 F F F
47382024-02-20T22:43:46.256Z 28 560-579 40 40 40 75 75 75 F F F
47392024-02-20T22:43:46.256Z 29 580-599 43 43 43 81 81 81 F F F
47402024-02-20T22:43:46.256ZMax gen: 43, Max flush: 81
47412024-02-20T22:43:46.256Z
47422024-02-20T22:43:46.256ZBegin loop 22
47432024-02-20T22:43:46.256ZDownstairs to restart: 0
47442024-02-20T22:43:46.269Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802089246Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311}
47452024-02-20T22:43:46.294Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802353207Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311}
47462024-02-20T22:43:46.294Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802391165Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311}
47472024-02-20T22:43:46.294Z{"msg":"Crucible dd51b2fc-eef3-4deb-be3f-4b23f93c5d04 has session id: 308a9eff-d03f-43ab-9827-6e36c3f96a81","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802424254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47482024-02-20T22:43:46.294Z{"msg":"Upstairs opts: Upstairs UUID: dd51b2fc-eef3-4deb-be3f-4b23f93c5d04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802463323Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47492024-02-20T22:43:46.294Z{"msg":"Crucible stats registered with UUID: dd51b2fc-eef3-4deb-be3f-4b23f93c5d04","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802504231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47502024-02-20T22:43:46.294ZCrucible runtime is spawned
47512024-02-20T22:43:46.294Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802668635Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311}
47522024-02-20T22:43:46.294Z{"msg":"dd51b2fc-eef3-4deb-be3f-4b23f93c5d04 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802763332Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47532024-02-20T22:43:46.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802791121Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47542024-02-20T22:43:46.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802856879Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47552024-02-20T22:43:46.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802905657Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47562024-02-20T22:43:46.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802931856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47572024-02-20T22:43:46.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802957505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47582024-02-20T22:43:46.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.802984294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47592024-02-20T22:43:46.294Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803007503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47602024-02-20T22:43:46.294Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803034012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47612024-02-20T22:43:46.295Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803061061Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47622024-02-20T22:43:46.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.80309668Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47632024-02-20T22:43:46.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803323222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47642024-02-20T22:43:46.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.80337898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47652024-02-20T22:43:46.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803774226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47662024-02-20T22:43:46.295Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803818224Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47672024-02-20T22:43:46.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803834034Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47682024-02-20T22:43:46.295Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803881312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47692024-02-20T22:43:46.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.803987748Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47702024-02-20T22:43:46.295Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.804093654Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47712024-02-20T22:43:46.295Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.804390324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47722024-02-20T22:43:46.295Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.804568677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47732024-02-20T22:43:46.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.804885936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47742024-02-20T22:43:46.295Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.804921765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47752024-02-20T22:43:46.295Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805017611Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47762024-02-20T22:43:46.295Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.80505912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47772024-02-20T22:43:46.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805088649Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47782024-02-20T22:43:46.296Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805112958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47792024-02-20T22:43:46.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805127707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47802024-02-20T22:43:46.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805387608Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47812024-02-20T22:43:46.296Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805414767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47822024-02-20T22:43:46.296Z{"msg":"[0]R flush_numbers[0..12]: [81, 79, 64, 66, 66, 81, 66, 64, 79, 66, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805434926Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47832024-02-20T22:43:46.296Z{"msg":"[0]R generation[0..12]: [43, 42, 35, 37, 37, 43, 37, 35, 42, 37, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805456685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47842024-02-20T22:43:46.296Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805471685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47852024-02-20T22:43:46.296Z{"msg":"[1]R flush_numbers[0..12]: [81, 79, 64, 66, 66, 81, 66, 64, 79, 66, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805493124Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47862024-02-20T22:43:46.296Z{"msg":"[1]R generation[0..12]: [43, 42, 35, 37, 37, 43, 37, 35, 42, 37, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805507364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47872024-02-20T22:43:46.296Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805528393Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47882024-02-20T22:43:46.297Z{"msg":"[2]R flush_numbers[0..12]: [81, 79, 64, 66, 66, 81, 66, 64, 79, 66, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805542642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47892024-02-20T22:43:46.297Z{"msg":"[2]R generation[0..12]: [43, 42, 35, 37, 37, 43, 37, 35, 42, 37, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805563242Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47902024-02-20T22:43:46.297Z{"msg":"Wait for a query_work_queue command to finish before sending IO
47912024-02-20T22:43:46.297Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805580531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47922024-02-20T22:43:46.297Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805593811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47932024-02-20T22:43:46.297Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.80560739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47942024-02-20T22:43:46.297Z{"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.80562109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47952024-02-20T22:43:46.297Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805634319Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47962024-02-20T22:43:46.297Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805654509Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47972024-02-20T22:43:46.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805667018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47982024-02-20T22:43:46.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805686947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
47992024-02-20T22:43:46.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805700547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
48002024-02-20T22:43:46.298Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805713736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
48012024-02-20T22:43:46.298Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805726516Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
48022024-02-20T22:43:46.298Z{"msg":"dd51b2fc-eef3-4deb-be3f-4b23f93c5d04 is now active with session: 308a9eff-d03f-43ab-9827-6e36c3f96a81","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805739666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
48032024-02-20T22:43:46.298Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805758105Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311,"session_id":"308a9eff-d03f-43ab-9827-6e36c3f96a81"}
48042024-02-20T22:43:46.298Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:45.805770804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311}
48052024-02-20T22:43:46.298ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48062024-02-20T22:43:46.298ZLoading write count information from file "/tmp/test_repair_verify.data"
48072024-02-20T22:43:46.298ZRun Repair workload
48082024-02-20T22:43:46.298Z01/30 Write block 110 len 20480 data: 6 7 8 9 7
48092024-02-20T22:43:46.778Z02/30 Write block 476 len 28672 data: 3 4 4 5 7 6 6
48102024-02-20T22:43:46.778Z03/30 Write block 185 len 32768 data: 6 5 5 5 5 4 3 2
48112024-02-20T22:43:46.778Z04/30 Read block 269 len 28672
48122024-02-20T22:43:46.778Z05/30 Flush
48132024-02-20T22:43:46.778Z06/30 Flush
48142024-02-20T22:43:46.778Z07/30 Write block 233 len 8192 data: 8 8
48152024-02-20T22:43:46.778Z08/30 Read block 273 len 12288
48162024-02-20T22:43:46.778Z09/30 Read block 573 len 12288
48172024-02-20T22:43:46.779Z10/30 Write block 64 len 36864 data: 2 2 2 2 3 4 5 4 4
48182024-02-20T22:43:46.779Z11/30 Read block 187 len 40960
48192024-02-20T22:43:46.779Z12/30 Read block 458 len 36864
48202024-02-20T22:43:46.779Z13/30 Write block 534 len 20480 data: 5 5 5 3 4
48212024-02-20T22:43:46.779Z14/30 Read block 60 len 32768
48222024-02-20T22:43:46.779Z15/30 Read block 173 len 36864
48232024-02-20T22:43:46.779Z16/30 Read block 302 len 24576
48242024-02-20T22:43:46.779Z17/30 Read block 373 len 4096
48252024-02-20T22:43:46.779Z18/30 Write block 363 len 4096 data: 10
48262024-02-20T22:43:46.779Z19/30 Write block 156 len 28672 data: 3 4 4 4 4 4 3
48272024-02-20T22:43:46.779Z20/30 Flush
48282024-02-20T22:43:46.779Z21/30 Write block 570 len 4096 data: 3
48292024-02-20T22:43:46.780Z22/30 Read block 547 len 8192
48302024-02-20T22:43:46.780Z23/30 Read block 237 len 40960
48312024-02-20T22:43:46.780Z24/30 Write block 322 len 8192 data: 8 5
48322024-02-20T22:43:46.780Z25/30 Read block 74 len 24576
48332024-02-20T22:43:46.780Z26/30 Read block 256 len 40960
48342024-02-20T22:43:46.780Z27/30 Read block 579 len 24576
48352024-02-20T22:43:46.780Z28/30 Read block 80 len 4096
48362024-02-20T22:43:46.780Z29/30 Write block 580 len 12288 data: 4 3 5
48372024-02-20T22:43:46.780Z30/30 Write block 139 len 28672 data: 5 5 4 5 6 5 4
48382024-02-20T22:43:46.780Z----------------------------------------------------------------
48392024-02-20T22:43:46.780Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48402024-02-20T22:43:46.780ZStates: Active Active Active
48412024-02-20T22:43:46.780ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48422024-02-20T22:43:46.780Z 1 Acked 1000 Write 5 Done Done Done false
48432024-02-20T22:43:46.781Z 2 Acked 1001 Write 7 Done Done Done false
48442024-02-20T22:43:46.781Z 3 Acked 1002 Write 8 Done Done Done false
48452024-02-20T22:43:46.781Z 4 Acked 1003 Read 7 Sent Done Done false
48462024-02-20T22:43:46.781Z 5 Acked 1004 Flush 0 Sent Done Done false
48472024-02-20T22:43:46.781Z 6 Acked 1005 Flush 0 Sent Done Done false
48482024-02-20T22:43:46.781Z 7 Acked 1006 Write 2 Sent Done Done false
48492024-02-20T22:43:46.781Z 8 Acked 1007 Read 3 Sent Done Done false
48502024-02-20T22:43:46.781Z 9 Acked 1008 Read 3 Sent Done Done false
48512024-02-20T22:43:46.781Z 10 Acked 1009 Write 9 Sent Done Done false
48522024-02-20T22:43:46.781Z 11 Acked 1010 Read 10 Sent Done Done false
48532024-02-20T22:43:46.781Z 12 Acked 1011 Read 9 Sent Done Done false
48542024-02-20T22:43:46.781Z 13 Acked 1012 Write 5 Sent Done Done false
48552024-02-20T22:43:46.781Z 14 Acked 1013 Read 8 Sent Done Done false
48562024-02-20T22:43:46.782Z 15 Acked 1014 Read 9 Sent Done Done false
48572024-02-20T22:43:46.782Z 16 Acked 1015 Read 6 Sent Done Done false
48582024-02-20T22:43:46.782Z 17 Acked 1016 Read 1 Sent Done Done false
48592024-02-20T22:43:46.782Z 18 Acked 1017 Write 1 Sent Done Done false
48602024-02-20T22:43:46.782Z 19 Acked 1018 Write 7 Sent Done Done false
48612024-02-20T22:43:46.782Z 20 Acked 1019 Flush 0 Sent Done Done false
48622024-02-20T22:43:46.782Z 21 Acked 1020 Write 1 Sent Done Done false
48632024-02-20T22:43:46.782Z 22 Acked 1021 Read 2 Sent Done Done false
48642024-02-20T22:43:46.782Z 23 Acked 1022 Read 10 Sent Done Done false
48652024-02-20T22:43:46.782Z 24 Acked 1023 Write 2 Sent Done Done false
48662024-02-20T22:43:46.782Z 25 Acked 1024 Read 6 Sent Done Done false
48672024-02-20T22:43:46.782Z 26 Acked 1025 Read 10 Sent Done Done false
48682024-02-20T22:43:46.782Z 27 Acked 1026 Read 6 Sent Done Done false
48692024-02-20T22:43:46.783Z 28 Acked 1027 Read 1 Sent Done Done false
48702024-02-20T22:43:46.783Z 29 Acked 1028 Write 3 Sent Done Done false
48712024-02-20T22:43:46.783Z 30 Acked 1029 Write 7 Sent Sent Sent false
48722024-02-20T22:43:46.783Z STATES DS:0 DS:1 DS:2 TOTAL
48732024-02-20T22:43:46.783Z New 0 0 0 0
48742024-02-20T22:43:46.783Z Sent 27 1 1 29
48752024-02-20T22:43:46.783Z Done 3 29 29 61
48762024-02-20T22:43:46.783Z Skipped 0 0 0 0
48772024-02-20T22:43:46.783Z Error 0 0 0 0
48782024-02-20T22:43:46.783ZLast Flush: 0 1019 1019
48792024-02-20T22:43:46.783ZDownstairs last five completed:
48802024-02-20T22:43:46.783ZUpstairs last five completed: 30 29 28 27 26
48812024-02-20T22:43:46.783Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:45.836170506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1311}
48822024-02-20T22:43:46.783ZWrote out file "/tmp/test_repair_verify.data"
48832024-02-20T22:43:46.783Z
48842024-02-20T22:43:46.783ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48852024-02-20T22:43:46.784ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48862024-02-20T22:43:46.784Z 0 000-019 43 43 43 81 81 81 F F F
48872024-02-20T22:43:46.784Z 1 020-039 42 42 42 79 79 79 F F F
48882024-02-20T22:43:46.784Z 2 040-059 35 35 35 64 64 64 F F F
48892024-02-20T22:43:46.784Z 3 060-079 37 44 44 66 84 84 F F F <---
48902024-02-20T22:43:46.784Z 4 080-099 37 37 37 66 66 66 F F F
48912024-02-20T22:43:46.784Z 5 100-119 43 44 44 81 82 82 T F F <---
48922024-02-20T22:43:46.784Z 6 120-139 37 37 37 66 66 66 F T T <---
48932024-02-20T22:43:46.784Z 7 140-159 35 44 44 64 84 84 F T T <---
48942024-02-20T22:43:46.784Z 8 160-179 42 44 44 79 84 84 F F F <---
48952024-02-20T22:43:46.784Z 9 180-199 37 44 44 66 82 82 T F F <---
48962024-02-20T22:43:46.784Z 10 200-219 41 41 41 76 76 76 F F F
48972024-02-20T22:43:46.784Z 11 220-239 43 44 44 81 84 84 F F F <---
48982024-02-20T22:43:46.784Z 12 240-259 41 41 41 76 76 76 F F F
48992024-02-20T22:43:46.784Z 13 260-279 43 43 43 81 81 81 F F F
49002024-02-20T22:43:46.784Z 14 280-299 38 38 38 71 71 71 F F F
49012024-02-20T22:43:46.784Z 15 300-319 42 42 42 80 80 80 F F F
49022024-02-20T22:43:46.784Z 16 320-339 43 43 43 81 81 81 F T T <---
49032024-02-20T22:43:46.784Z 17 340-359 43 43 43 81 81 81 F F F
49042024-02-20T22:43:46.784Z 18 360-379 42 44 44 79 84 84 F F F <---
49052024-02-20T22:43:46.784Z 19 380-399 42 42 42 77 77 77 F F F
49062024-02-20T22:43:46.784Z 20 400-419 42 42 42 77 77 77 F F F
49072024-02-20T22:43:46.784Z 21 420-439 41 41 41 76 76 76 F F F
49082024-02-20T22:43:46.785Z 22 440-459 43 43 43 81 81 81 F F F
49092024-02-20T22:43:46.796Z 23 460-479 41 44 44 76 82 82 T F F <---
49102024-02-20T22:43:46.796Z 24 480-499 43 44 44 81 82 82 T F F <---
49112024-02-20T22:43:46.796Z 25 500-519 39 39 39 74 74 74 F F F
49122024-02-20T22:43:46.796Z 26 520-539 38 44 44 68 84 84 F F F <---
49132024-02-20T22:43:46.797Z 27 540-559 42 42 42 77 77 77 F F F
49142024-02-20T22:43:46.797Z 28 560-579 40 40 40 75 75 75 F T T <---
49152024-02-20T22:43:46.797Z 29 580-599 43 43 43 81 81 81 F T T <---
49162024-02-20T22:43:46.797ZMax gen: 44, Max flush: 84
49172024-02-20T22:43:46.797ZError: Difference in extent metadata found!
49182024-02-20T22:43:46.797ZA Difference in extent metadata is expected here
49192024-02-20T22:43:46.797ZOn loop 22
49202024-02-20T22:43:46.797Z
49212024-02-20T22:43:46.797ZVerifying data now
49222024-02-20T22:43:51.997ZNo change after verify
49232024-02-20T22:43:51.997ZLoop: 22 Downstairs dump after verify (and repair):
49242024-02-20T22:43:52.010ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49252024-02-20T22:43:52.010Z 0 000-019 43 43 43 81 81 81 F F F
49262024-02-20T22:43:52.010Z 1 020-039 42 42 42 79 79 79 F F F
49272024-02-20T22:43:52.010Z 2 040-059 35 35 35 64 64 64 F F F
49282024-02-20T22:43:52.010Z 3 060-079 44 44 44 84 84 84 F F F
49292024-02-20T22:43:52.010Z 4 080-099 37 37 37 66 66 66 F F F
49302024-02-20T22:43:52.010Z 5 100-119 44 44 44 82 82 82 F F F
49312024-02-20T22:43:52.010Z 6 120-139 45 45 45 85 85 85 F F F
49322024-02-20T22:43:52.015Z 7 140-159 45 45 45 85 85 85 F F F
49332024-02-20T22:43:52.015Z 8 160-179 44 44 44 84 84 84 F F F
49342024-02-20T22:43:52.015Z 9 180-199 44 44 44 82 82 82 F F F
49352024-02-20T22:43:52.015Z 10 200-219 41 41 41 76 76 76 F F F
49362024-02-20T22:43:52.015Z 11 220-239 44 44 44 84 84 84 F F F
49372024-02-20T22:43:52.015Z 12 240-259 41 41 41 76 76 76 F F F
49382024-02-20T22:43:52.015Z 13 260-279 43 43 43 81 81 81 F F F
49392024-02-20T22:43:52.015Z 14 280-299 38 38 38 71 71 71 F F F
49402024-02-20T22:43:52.015Z 15 300-319 42 42 42 80 80 80 F F F
49412024-02-20T22:43:52.015Z 16 320-339 45 45 45 85 85 85 F F F
49422024-02-20T22:43:52.015Z 17 340-359 43 43 43 81 81 81 F F F
49432024-02-20T22:43:52.015Z 18 360-379 44 44 44 84 84 84 F F F
49442024-02-20T22:43:52.016Z 19 380-399 42 42 42 77 77 77 F F F
49452024-02-20T22:43:52.016Z 20 400-419 42 42 42 77 77 77 F F F
49462024-02-20T22:43:52.016Z 21 420-439 41 41 41 76 76 76 F F F
49472024-02-20T22:43:52.016Z 22 440-459 43 43 43 81 81 81 F F F
49482024-02-20T22:43:52.016Z 23 460-479 44 44 44 82 82 82 F F F
49492024-02-20T22:43:52.016Z 24 480-499 44 44 44 82 82 82 F F F
49502024-02-20T22:43:52.016Z 25 500-519 39 39 39 74 74 74 F F F
49512024-02-20T22:43:52.016Z 26 520-539 44 44 44 84 84 84 F F F
49522024-02-20T22:43:52.016Z 27 540-559 42 42 42 77 77 77 F F F
49532024-02-20T22:43:52.016Z 28 560-579 45 45 45 85 85 85 F F F
49542024-02-20T22:43:52.016Z 29 580-599 45 45 45 85 85 85 F F F
49552024-02-20T22:43:52.016ZMax gen: 45, Max flush: 85
49562024-02-20T22:43:52.016Z
49572024-02-20T22:43:52.017ZBegin loop 23
49582024-02-20T22:43:52.017ZDownstairs to restart: 2
49592024-02-20T22:43:52.031Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.564522459Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319}
49602024-02-20T22:43:52.040Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.564858027Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319}
49612024-02-20T22:43:52.040Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
49622024-02-20T22:43:52.040Z,"time":"2024-02-20T22:43:51.564927864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319}
49632024-02-20T22:43:52.040Z{"msg":"Crucible 182527e0-5c22-41d6-82e5-7c62defec1f4 has session id: cc22fe97-642d-43dd-828d-835b81480c19","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.564962883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49642024-02-20T22:43:52.040Z{"msg":"Upstairs opts: Upstairs UUID: 182527e0-5c22-41d6-82e5-7c62defec1f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565005401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49652024-02-20T22:43:52.040Z{"msg":"Crucible stats registered with UUID: 182527e0-5c22-41d6-82e5-7c62defec1f4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56504034Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49662024-02-20T22:43:52.040Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565073109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319}
49672024-02-20T22:43:52.040Z{"msg":"182527e0-5c22-41d6-82e5-7c62defec1f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565100028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49682024-02-20T22:43:52.040Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565116977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49692024-02-20T22:43:52.040Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565147756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49702024-02-20T22:43:52.040Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565164096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49712024-02-20T22:43:52.040Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565195904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49722024-02-20T22:43:52.040Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565212254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49732024-02-20T22:43:52.041Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565243583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49742024-02-20T22:43:52.041Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565259382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49752024-02-20T22:43:52.041Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565295141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49762024-02-20T22:43:52.041Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56531229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49772024-02-20T22:43:52.041Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565426416Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49782024-02-20T22:43:52.041Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565617789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49792024-02-20T22:43:52.041Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.565683107Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49802024-02-20T22:43:52.041Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566104682Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49812024-02-20T22:43:52.041Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56614231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49822024-02-20T22:43:52.041Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56616427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49832024-02-20T22:43:52.041Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566186569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49842024-02-20T22:43:52.041Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566200908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49852024-02-20T22:43:52.041Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566221438Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49862024-02-20T22:43:52.041Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566795817Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49872024-02-20T22:43:52.041Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566880294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49882024-02-20T22:43:52.041Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.566931662Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49892024-02-20T22:43:52.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567349277Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49902024-02-20T22:43:52.041Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567405095Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49912024-02-20T22:43:52.041Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567428284Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49922024-02-20T22:43:52.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567451604Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49932024-02-20T22:43:52.041Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567467673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49942024-02-20T22:43:52.041Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567489802Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49952024-02-20T22:43:52.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567502912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49962024-02-20T22:43:52.041Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56753784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49972024-02-20T22:43:52.042Z{"msg":"[0]R flush_numbers[0..12]: [81, 79, 64, 84, 66, 82, 85, 85, 84, 82, 76, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56755323Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49982024-02-20T22:43:52.042Z{"msg":"[0]R generation[0..12]: [43, 42, 35, 44, 37, 44, 45, 45, 44, 44, 41, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567574419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
49992024-02-20T22:43:52.042Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567589199Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50002024-02-20T22:43:52.042Z{"msg":"[1]R flush_numbers[0..12]: [81, 79, 64, 84, 66, 82, 85, 85, 84, 82, 76, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567610328Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50012024-02-20T22:43:52.042Z{"msg":"[1]R generation[0..12]: [43, 42, 35, 44, 37, 44, 45, 45, 44, 44, 41, 44]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
50022024-02-20T22:43:52.042Z2024-02-20T22:43:51.567624628Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50032024-02-20T22:43:52.042Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567651476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50042024-02-20T22:43:52.042Z{"msg":"[2]R flush_numbers[0..12]: [81, 79, 64, 84, 66, 82, 85, 85, 84, 82, 76, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567673926Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50052024-02-20T22:43:52.042Z{"msg":"[2]R generation[0..12]: [43, 42, 35, 44, 37, 44, 45, 45, 44, 44, 41, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567706885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50062024-02-20T22:43:52.042Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567728544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50072024-02-20T22:43:52.042Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567751333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50082024-02-20T22:43:52.042Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567765732Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50092024-02-20T22:43:52.042Z{"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567785362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50102024-02-20T22:43:52.042Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567799081Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50112024-02-20T22:43:52.042Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.56783136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50122024-02-20T22:43:52.042Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567855269Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50132024-02-20T22:43:52.042Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567869199Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50142024-02-20T22:43:52.042Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567882698Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50152024-02-20T22:43:52.042Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567908467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50162024-02-20T22:43:52.042Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567937126Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50172024-02-20T22:43:52.042Z{"msg":"182527e0-5c22-41d6-82e5-7c62defec1f4 is now active with session: cc22fe97-642d-43dd-828d-835b81480c19","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567951636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50182024-02-20T22:43:52.042Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567970465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319,"session_id":"cc22fe97-642d-43dd-828d-835b81480c19"}
50192024-02-20T22:43:52.042Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:51.567987164Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319}
50202024-02-20T22:43:52.043ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50212024-02-20T22:43:52.043ZLoading write count information from file "/tmp/test_repair_verify.data"
50222024-02-20T22:43:52.043ZRun Repair workload
50232024-02-20T22:43:52.043Z01/30 Write block 551 len 28672 data: 4 4 3 3 3 3 3
50242024-02-20T22:43:52.047Z02/30 Write block 52 len 20480 data: 5 4 4 4 3
50252024-02-20T22:43:52.047Z03/30 Write block 402 len 24576 data: 6 6 6 6 7 7
50262024-02-20T22:43:52.047Z04/30 Write block 191 len 32768 data: 4 3 3 3 4 4 4 3
50272024-02-20T22:43:52.047Z05/30 Write block 506 len 24576 data: 4 4 5 5 7 7
50282024-02-20T22:43:52.047Z06/30 Read block 136 len 8192
50292024-02-20T22:43:52.047Z07/30 Flush
50302024-02-20T22:43:52.047Z08/30 Write block 22 len 40960 data: 4 3 2 6 6 10 11 13 12 11
50312024-02-20T22:43:52.047Z09/30 Read block 459 len 12288
50322024-02-20T22:43:52.048Z10/30 Write block 18 len 12288 data: 5 4 4
50332024-02-20T22:43:52.048Z11/30 Write block 440 len 32768 data: 5 5 6 7 6 7 7 6
50342024-02-20T22:43:52.048Z12/30 Read block 386 len 12288
50352024-02-20T22:43:52.051Z13/30 Flush
50362024-02-20T22:43:52.051Z14/30 Write block 454 len 8192 data: 6 7
50372024-02-20T22:43:52.051Z15/30 Write block 228 len 12288 data: 5 6 8
50382024-02-20T22:43:52.051Z16/30 Write block 293 len 28672 data: 2 3 2 2 2 2 2
50392024-02-20T22:43:52.051Z17/30 Read block 498 len 8192
50402024-02-20T22:43:52.057Z18/30 Write block 551 len 24576 data: 5 5 4 4 4 4
50412024-02-20T22:43:52.057Z19/30 Write block 106 len 24576 data: 2 2 4 5 7 8
50422024-02-20T22:43:52.057Z20/30 Read block 346 len 24576
50432024-02-20T22:43:52.057Z21/30 Write block 527 len 4096 data: 6
50442024-02-20T22:43:52.057Z22/30 Read block 479 len 40960
50452024-02-20T22:43:52.063Z23/30 Read block 234 len 32768
50462024-02-20T22:43:52.063Z24/30 Read block 77 len 4096
50472024-02-20T22:43:52.063Z25/30 Write block 306 len 36864 data: 5 5 4 5 5 5 5 5 5
50482024-02-20T22:43:52.063Z26/30 Write block 287 len 20480 data: 5 4 4 3 3
50492024-02-20T22:43:52.063Z27/30 Write block 31 len 24576 data: 12 11 11 8 8 6
50502024-02-20T22:43:52.063Z28/30 Read block 245 len 4096
50512024-02-20T22:43:52.076Z29/30 Write block 257 len 24576 data: 2 2 2 2 2 2
50522024-02-20T22:43:52.076Z30/30 Write block 175 len 32768 data: 5 5 5 5 7 8 7 7
50532024-02-20T22:43:52.076Z----------------------------------------------------------------
50542024-02-20T22:43:52.077Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50552024-02-20T22:43:52.077ZStates: Active Active Active
50562024-02-20T22:43:52.077ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50572024-02-20T22:43:52.077Z 1 Acked 1000 Write 7 Done Done Done false
50582024-02-20T22:43:52.077Z 2 Acked 1001 Write 5 Done Done Done false
50592024-02-20T22:43:52.078Z 3 Acked 1002 Write 6 Done Done Done false
50602024-02-20T22:43:52.078Z 4 Acked 1003 Write 8 Done Done Done false
50612024-02-20T22:43:52.078Z 5 Acked 1004 Write 6 Done Done Done false
50622024-02-20T22:43:52.078Z 6 Acked 1005 Read 2 Done Done Sent false
50632024-02-20T22:43:52.078Z 7 Acked 1006 Flush 0 Done Done Sent false
50642024-02-20T22:43:52.078Z 8 Acked 1007 Write 10 Done Done Sent false
50652024-02-20T22:43:52.078Z 9 Acked 1008 Read 3 Done Done Sent false
50662024-02-20T22:43:52.078Z 10 Acked 1009 Write 3 Done Done Sent false
50672024-02-20T22:43:52.078Z 11 Acked 1010 Write 8 Done Done Sent false
50682024-02-20T22:43:52.079Z 12 Acked 1011 Read 3 Done Done Sent false
50692024-02-20T22:43:52.079Z 13 Acked 1012 Flush 0 Done Done Sent false
50702024-02-20T22:43:52.079Z 14 Acked 1013 Write 2 Done Done Sent false
50712024-02-20T22:43:52.079Z 15 Acked 1014 Write 3 Done Done Sent false
50722024-02-20T22:43:52.079Z 16 Acked 1015 Write 7 Done Done Sent false
50732024-02-20T22:43:52.079Z 17 Acked 1016 Read 2 Done Done Sent false
50742024-02-20T22:43:52.079Z 18 Acked 1017 Write 6 Done Done Sent false
50752024-02-20T22:43:52.080Z 19 Acked 1018 Write 6 Done Done Sent false
50762024-02-20T22:43:52.080Z 20 Acked 1019 Read 6 Done Done Sent false
50772024-02-20T22:43:52.080Z 21 Acked 1020 Write 1 Done Done Sent false
50782024-02-20T22:43:52.080Z 22 Acked 1021 Read 10 Done Done Sent false
50792024-02-20T22:43:52.080Z 23 Acked 1022 Read 8 Done Done Sent false
50802024-02-20T22:43:52.080Z 24 Acked 1023 Read 1 Done Done Sent false
50812024-02-20T22:43:52.080Z 25 Acked 1024 Write 9 Done Done Sent false
50822024-02-20T22:43:52.081Z 26 Acked 1025 Write 5 Done Done Sent false
50832024-02-20T22:43:52.081Z 27 Acked 1026 Write 6 Done Done Sent false
50842024-02-20T22:43:52.081Z 28 Acked 1027 Read 1 Done Done Sent false
50852024-02-20T22:43:52.081Z 29 Acked 1028 Write 6 Done Done Sent false
50862024-02-20T22:43:52.081Z 30 Acked 1029 Write 8 Sent Sent Sent false
50872024-02-20T22:43:52.081Z STATES DS:0 DS:1 DS:2 TOTAL
50882024-02-20T22:43:52.082Z New 0 0 0 0
50892024-02-20T22:43:52.082Z Sent 1 1 25 27
50902024-02-20T22:43:52.082Z Done 29 29 5 63
50912024-02-20T22:43:52.082Z Skipped 0 0 0 0
50922024-02-20T22:43:52.082Z Error 0 0 0 0
50932024-02-20T22:43:52.082ZLast Flush: 1012 1012 0
50942024-02-20T22:43:52.083ZDownstairs last five completed:
50952024-02-20T22:43:52.083ZUpstairs last five completed: 30 29 28 27 26
50962024-02-20T22:43:52.083Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:51.597228958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1319}
50972024-02-20T22:43:52.083ZWrote out file "/tmp/test_repair_verify.data"
50982024-02-20T22:43:52.083Z
50992024-02-20T22:43:52.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51002024-02-20T22:43:52.099ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51012024-02-20T22:43:52.099Z 0 000-019 46 46 43 87 87 81 F F F <---
51022024-02-20T22:43:52.099Z 1 020-039 46 46 42 87 87 79 T T F <---
51032024-02-20T22:43:52.099Z 2 040-059 46 46 35 86 86 64 F F T <---
51042024-02-20T22:43:52.099Z 3 060-079 44 44 44 84 84 84 F F F
51052024-02-20T22:43:52.099Z 4 080-099 37 37 37 66 66 66 F F F
51062024-02-20T22:43:52.099Z 5 100-119 44 44 44 82 82 82 T T F <---
51072024-02-20T22:43:52.099Z 6 120-139 45 45 45 85 85 85 F F F
51082024-02-20T22:43:52.099Z 7 140-159 45 45 45 85 85 85 F F F
51092024-02-20T22:43:52.099Z 8 160-179 44 44 44 84 84 84 T T F <---
51102024-02-20T22:43:52.099Z 9 180-199 46 46 44 86 86 82 T T T <---
51112024-02-20T22:43:52.100Z 10 200-219 41 41 41 76 76 76 F F F
51122024-02-20T22:43:52.100Z 11 220-239 44 44 44 84 84 84 T T F <---
51132024-02-20T22:43:52.100Z 12 240-259 41 41 41 76 76 76 T T F <---
51142024-02-20T22:43:52.100Z 13 260-279 43 43 43 81 81 81 T T F <---
51152024-02-20T22:43:52.100Z 14 280-299 38 38 38 71 71 71 T T F <---
51162024-02-20T22:43:52.100Z 15 300-319 42 42 42 80 80 80 T T F <---
51172024-02-20T22:43:52.100Z 16 320-339 45 45 45 85 85 85 F F F
51182024-02-20T22:43:52.100Z 17 340-359 43 43 43 81 81 81 F F F
51192024-02-20T22:43:52.100Z 18 360-379 44 44 44 84 84 84 F F F
51202024-02-20T22:43:52.100Z 19 380-399 42 42 42 77 77 77 F F F
51212024-02-20T22:43:52.100Z 20 400-419 46 46 42 86 86 77 F F T <---
51222024-02-20T22:43:52.100Z 21 420-439 41 41 41 76 76 76 F F F
51232024-02-20T22:43:52.100Z 22 440-459 46 46 43 87 87 81 T T F <---
51242024-02-20T22:43:52.100Z 23 460-479 44 44 44 82 82 82 F F F
51252024-02-20T22:43:52.100Z 24 480-499 44 44 44 82 82 82 F F F
51262024-02-20T22:43:52.100Z 25 500-519 46 46 39 86 86 74 F F T <---
51272024-02-20T22:43:52.100Z 26 520-539 44 44 44 84 84 84 T T F <---
51282024-02-20T22:43:52.100Z 27 540-559 46 46 42 86 86 77 T T T <---
51292024-02-20T22:43:52.100Z 28 560-579 45 45 45 85 85 85 F F F
51302024-02-20T22:43:52.100Z 29 580-599 45 45 45 85 85 85 F F F
51312024-02-20T22:43:52.100ZMax gen: 46, Max flush: 87
51322024-02-20T22:43:52.100ZError: Difference in extent metadata found!
51332024-02-20T22:43:52.124ZA Difference in extent metadata is expected here
51342024-02-20T22:43:52.124ZOn loop 23
51352024-02-20T22:43:52.124Z
51362024-02-20T22:43:52.124ZVerifying data now
51372024-02-20T22:43:57.875ZNo change after verify
51382024-02-20T22:43:57.875ZLoop: 23 Downstairs dump after verify (and repair):
51392024-02-20T22:43:57.889ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51402024-02-20T22:43:57.889Z 0 000-019 46 46 46 87 87 87 F F F
51412024-02-20T22:43:57.889Z 1 020-039 47 47 47 88 88 88 F F F
51422024-02-20T22:43:57.889Z 2 040-059 46 46 46 86 86 86 F F F
51432024-02-20T22:43:57.889Z 3 060-079 44 44 44 84 84 84 F F F
51442024-02-20T22:43:57.889Z 4 080-099 37 37 37 66 66 66 F F F
51452024-02-20T22:43:57.889Z 5 100-119 47 47 47 88 88 88 F F F
51462024-02-20T22:43:57.889Z 6 120-139 45 45 45 85 85 85 F F F
51472024-02-20T22:43:57.889Z 7 140-159 45 45 45 85 85 85 F F F
51482024-02-20T22:43:57.890Z 8 160-179 47 47 47 88 88 88 F F F
51492024-02-20T22:43:57.890Z 9 180-199 47 47 47 88 88 88 F F F
51502024-02-20T22:43:57.895Z 10 200-219 41 41 41 76 76 76 F F F
51512024-02-20T22:43:57.895Z 11 220-239 47 47 47 88 88 88 F F F
51522024-02-20T22:43:57.895Z 12 240-259 47 47 47 88 88 88 F F F
51532024-02-20T22:43:57.895Z 13 260-279 47 47 47 88 88 88 F F F
51542024-02-20T22:43:57.895Z 14 280-299 47 47 47 88 88 88 F F F
51552024-02-20T22:43:57.895Z 15 300-319 47 47 47 88 88 88 F F F
51562024-02-20T22:43:57.895Z 16 320-339 45 45 45 85 85 85 F F F
51572024-02-20T22:43:57.895Z 17 340-359 43 43 43 81 81 81 F F F
51582024-02-20T22:43:57.895Z 18 360-379 44 44 44 84 84 84 F F F
51592024-02-20T22:43:57.895Z 19 380-399 42 42 42 77 77 77 F F F
51602024-02-20T22:43:57.895Z 20 400-419 46 46 46 86 86 86 F F F
51612024-02-20T22:43:57.895Z 21 420-439 41 41 41 76 76 76 F F F
51622024-02-20T22:43:57.895Z 22 440-459 47 47 47 88 88 88 F F F
51632024-02-20T22:43:57.895Z 23 460-479 44 44 44 82 82 82 F F F
51642024-02-20T22:43:57.896Z 24 480-499 44 44 44 82 82 82 F F F
51652024-02-20T22:43:57.896Z 25 500-519 46 46 46 86 86 86 F F F
51662024-02-20T22:43:57.896Z 26 520-539 47 47 47 88 88 88 F F F
51672024-02-20T22:43:57.896Z 27 540-559 47 47 47 88 88 88 F F F
51682024-02-20T22:43:57.896Z 28 560-579 45 45 45 85 85 85 F F F
51692024-02-20T22:43:57.896Z 29 580-599 45 45 45 85 85 85 F F F
51702024-02-20T22:43:57.896ZMax gen: 47, Max flush: 88
51712024-02-20T22:43:57.896Z
51722024-02-20T22:43:57.896ZBegin loop 24
51732024-02-20T22:43:57.896ZDownstairs to restart: 1
51742024-02-20T22:43:57.912Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44521882Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327}
51752024-02-20T22:43:57.924Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445488731Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327}
51762024-02-20T22:43:57.925Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445526499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327}
51772024-02-20T22:43:57.925Z{"msg":"Crucible 5ed3ef7d-ca50-4441-9fb5-95abf3cd3682 has session id: 1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5","v":0,"name":"crucible","level":30Crucible runtime is spawned
51782024-02-20T22:43:57.925Z,"time":"2024-02-20T22:43:57.445550108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51792024-02-20T22:43:57.925Z{"msg":"Upstairs opts: Upstairs UUID: 5ed3ef7d-ca50-4441-9fb5-95abf3cd3682, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445589817Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51802024-02-20T22:43:57.925Z{"msg":"Crucible stats registered with UUID: 5ed3ef7d-ca50-4441-9fb5-95abf3cd3682","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445633405Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51812024-02-20T22:43:57.925Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445667594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327}
51822024-02-20T22:43:57.925Z{"msg":"5ed3ef7d-ca50-4441-9fb5-95abf3cd3682 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445759081Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51832024-02-20T22:43:57.925Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44579503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51842024-02-20T22:43:57.925Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445812989Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51852024-02-20T22:43:57.925Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445829798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51862024-02-20T22:43:57.925Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445873547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51872024-02-20T22:43:57.925Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445915226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51882024-02-20T22:43:57.925Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445943214Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51892024-02-20T22:43:57.925Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.445980723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51902024-02-20T22:43:57.925Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.446016072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51912024-02-20T22:43:57.925Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44605249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51922024-02-20T22:43:57.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.446212895Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51932024-02-20T22:43:57.926Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.446264293Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51942024-02-20T22:43:57.926Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.446296392Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51952024-02-20T22:43:57.926Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.446966768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51962024-02-20T22:43:57.926Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447021516Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51972024-02-20T22:43:57.926Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447055785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51982024-02-20T22:43:57.926Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447072134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
51992024-02-20T22:43:57.926Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447131582Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52002024-02-20T22:43:57.926Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44716987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52012024-02-20T22:43:57.926Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447593245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52022024-02-20T22:43:57.926Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447634864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52032024-02-20T22:43:57.926Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.447677922Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52042024-02-20T22:43:57.926Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448116907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52052024-02-20T22:43:57.926Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448159775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52062024-02-20T22:43:57.926Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448177394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52072024-02-20T22:43:57.926Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448215193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52082024-02-20T22:43:57.926Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448242422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52092024-02-20T22:43:57.926Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448260462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52102024-02-20T22:43:57.926Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448273651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52112024-02-20T22:43:57.926Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44829504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52122024-02-20T22:43:57.926Z{"msg":"[0]R flush_numbers[0..12]: [87, 88, 86, 84, 66, 88, 85, 85, 88, 88, 76, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44831053Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52132024-02-20T22:43:57.926Z{"msg":"[0]R generation[0..12]: [46, 47, 46, 44, 37, 47, 45, 45, 47, 47, 41, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448325299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52142024-02-20T22:43:57.926Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448339809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52152024-02-20T22:43:57.926Z{"msg":"[1]R flush_numbers[0..12]: [87, 88, 86, 84, 66, 88, 85, 85, 88, 88, 76, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448360818Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52162024-02-20T22:43:57.927Z{"msg":"[1]R generation[0..12]: [46, 47, 46, 44, 37, 47, 45, 45, 47, 47, 41, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448374937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52172024-02-20T22:43:57.927Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448389147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52182024-02-20T22:43:57.927Z{"msg":"[2]R flush_numbers[0..12]: [87, 88, 86, 84, 66, 88, 85, 85, 88, 88, 76, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448403376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52192024-02-20T22:43:57.927Z{"msg":"[2]R generation[0..12]: [46, 47, 46, 44, 37, 47, 45, 45, 47, 47, 41, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448425096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52202024-02-20T22:43:57.927Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448439895Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52212024-02-20T22:43:57.927Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448465454Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52222024-02-20T22:43:57.927Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448479414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52232024-02-20T22:43:57.927Z{"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448492363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52242024-02-20T22:43:57.927ZWait for a query_work_queue command to finish before sending IO
52252024-02-20T22:43:57.927Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448512073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52262024-02-20T22:43:57.927Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448525592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52272024-02-20T22:43:57.927Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448537821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52282024-02-20T22:43:57.927Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448551001Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52292024-02-20T22:43:57.927Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44857168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52302024-02-20T22:43:57.927Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.44858528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52312024-02-20T22:43:57.927Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448597559Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52322024-02-20T22:43:57.927Z{"msg":"5ed3ef7d-ca50-4441-9fb5-95abf3cd3682 is now active with session: 1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448617679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52332024-02-20T22:43:57.927Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448630018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327,"session_id":"1880b5d1-cb3c-49e4-8f56-f8a8d91e05b5"}
52342024-02-20T22:43:57.927Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:43:57.448642428Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327}
52352024-02-20T22:43:57.927ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52362024-02-20T22:43:57.927ZLoading write count information from file "/tmp/test_repair_verify.data"
52372024-02-20T22:43:57.927ZRun Repair workload
52382024-02-20T22:43:57.927Z01/30 Write block 194 len 12288 data: 4 5 5
52392024-02-20T22:43:57.928Z02/30 Read block 250 len 40960
52402024-02-20T22:43:57.954Z03/30 Write block 572 len 40960 data: 2 2 2 3 5 5 5 5 5 4
52412024-02-20T22:43:57.954Z04/30 Read block 29 len 32768
52422024-02-20T22:43:57.954Z05/30 Read block 592 len 12288
52432024-02-20T22:43:57.954Z06/30 Write block 469 len 28672 data: 8 5 3 2 2 2 3
52442024-02-20T22:43:57.954Z07/30 Write block 236 len 8192 data: 5 5
52452024-02-20T22:43:57.954Z08/30 Write block 564 len 28672 data: 4 3 4 4 4 3 4
52462024-02-20T22:43:57.954Z09/30 Read block 208 len 32768
52472024-02-20T22:43:57.954Z10/30 Read block 85 len 12288
52482024-02-20T22:43:57.955Z11/30 Write block 534 len 20480 data: 6 6 6 4 5
52492024-02-20T22:43:57.955Z12/30 Read block 127 len 4096
52502024-02-20T22:43:57.955Z13/30 Flush
52512024-02-20T22:43:57.955Z14/30 Write block 396 len 8192 data: 7 9
52522024-02-20T22:43:57.955Z15/30 Write block 533 len 28672 data: 4 7 7 7 5 6 4
52532024-02-20T22:43:57.955Z16/30 Read block 576 len 8192
52542024-02-20T22:43:57.956Z17/30 Read block 161 len 12288
52552024-02-20T22:43:57.956Z18/30 Read block 361 len 16384
52562024-02-20T22:43:57.956Z19/30 Write block 496 len 16384 data: 7 8 7 6
52572024-02-20T22:43:57.956Z20/30 Flush
52582024-02-20T22:43:57.956Z21/30 Write block 514 len 16384 data: 3 4 3 3
52592024-02-20T22:43:57.956Z22/30 Write block 474 len 40960 data: 3 4 4 5 5 6 8 7 7 7
52602024-02-20T22:43:57.956Z23/30 Write block 281 len 36864 data: 5 5 6 6 5 6 6 5 5
52612024-02-20T22:43:57.957Z24/30 Read block 19 len 32768
52622024-02-20T22:43:57.957Z25/30 Flush
52632024-02-20T22:43:57.957Z26/30 Write block 437 len 24576 data: 2 3 5 6 6 7
52642024-02-20T22:43:57.957Z27/30 Read block 353 len 8192
52652024-02-20T22:43:57.958Z28/30 Write block 540 len 32768 data: 4 4 4 4 3 3 2 4
52662024-02-20T22:43:57.958Z29/30 Read block 419 len 36864
52672024-02-20T22:43:57.958Z30/30 Read block 182 len 40960
52682024-02-20T22:43:57.984Z----------------------------------------------------------------
52692024-02-20T22:43:57.984Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52702024-02-20T22:43:57.984ZStates: Active Active Active
52712024-02-20T22:43:57.984ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52722024-02-20T22:43:57.984Z 1 Acked 1000 Write 3 Done Sent Done false
52732024-02-20T22:43:57.984Z 2 Acked 1001 Read 10 Done Sent Done false
52742024-02-20T22:43:57.984Z 3 Acked 1002 Write 10 Done Sent Done false
52752024-02-20T22:43:57.984Z 4 Acked 1003 Read 8 Done Sent Done false
52762024-02-20T22:43:57.984Z 5 Acked 1004 Read 3 Done Sent Done false
52772024-02-20T22:43:57.984Z 6 Acked 1005 Write 7 Done Sent Done false
52782024-02-20T22:43:57.985Z 7 Acked 1006 Write 2 Done Sent Done false
52792024-02-20T22:43:57.985Z 8 Acked 1007 Write 7 Done Sent Done false
52802024-02-20T22:43:57.985Z 9 Acked 1008 Read 8 Done Sent Done false
52812024-02-20T22:43:57.985Z 10 Acked 1009 Read 3 Done Sent Done false
52822024-02-20T22:43:57.985Z 11 Acked 1010 Write 5 Done Sent Done false
52832024-02-20T22:43:57.985Z 12 Acked 1011 Read 1 Done Sent Done false
52842024-02-20T22:43:57.985Z 13 Acked 1012 Flush 0 Done Sent Done false
52852024-02-20T22:43:57.985Z 14 Acked 1013 Write 2 Done Sent Done false
52862024-02-20T22:43:57.985Z 15 Acked 1014 Write 7 Done Sent Done false
52872024-02-20T22:43:57.985Z 16 Acked 1015 Read 2 Done Sent Done false
52882024-02-20T22:43:57.985Z 17 Acked 1016 Read 3 Done Sent Done false
52892024-02-20T22:43:57.986Z 18 Acked 1017 Read 4 Done Sent Done false
52902024-02-20T22:43:57.986Z 19 Acked 1018 Write 4 Done Sent Done false
52912024-02-20T22:43:57.986Z 20 Acked 1019 Flush 0 Done Sent Done false
52922024-02-20T22:43:57.986Z 21 Acked 1020 Write 4 Done Sent Done false
52932024-02-20T22:43:57.986Z 22 Acked 1021 Write 10 Done Sent Done false
52942024-02-20T22:43:57.986Z 23 Acked 1022 Write 9 Done Sent Done false
52952024-02-20T22:43:57.986Z 24 Acked 1023 Read 8 Done Sent Done false
52962024-02-20T22:43:57.987Z 25 Acked 1024 Flush 0 Done Sent Done false
52972024-02-20T22:43:57.987Z 26 Acked 1025 Write 6 Done Sent Done false
52982024-02-20T22:43:57.987Z 27 Acked 1026 Read 2 Done Sent Done false
52992024-02-20T22:43:57.987Z 28 Acked 1027 Write 8 Done Sent Done false
53002024-02-20T22:43:57.987Z 29 Acked 1028 Read 9 Done Sent Done false
53012024-02-20T22:43:57.987Z 30 Acked 1029 Read 10 Done Sent Done false
53022024-02-20T22:43:57.987Z STATES DS:0 DS:1 DS:2 TOTAL
53032024-02-20T22:43:57.987Z New 0 0 0 0
53042024-02-20T22:43:57.987Z Sent 0 30 0 30
53052024-02-20T22:43:57.987Z Done 30 0 30 60
53062024-02-20T22:43:57.987Z Skipped 0 0 0 0
53072024-02-20T22:43:57.987Z Error 0 0 0 0
53082024-02-20T22:43:57.987ZLast Flush: 1024 0 1024
53092024-02-20T22:43:57.987ZDownstairs last five completed:
53102024-02-20T22:43:57.987ZUpstairs last five completed: 30 29 28 27 26
53112024-02-20T22:43:57.987Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:43:57.48183179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1327}
53122024-02-20T22:43:57.987ZWrote out file "/tmp/test_repair_verify.data"
53132024-02-20T22:43:57.987Z
53142024-02-20T22:43:57.987ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53152024-02-20T22:43:58.413ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53162024-02-20T22:43:58.413Z 0 000-019 46 46 46 87 87 87 F F F
53172024-02-20T22:43:58.413Z 1 020-039 47 47 47 88 88 88 F F F
53182024-02-20T22:43:58.414Z 2 040-059 46 46 46 86 86 86 F F F
53192024-02-20T22:43:58.414Z 3 060-079 44 44 44 84 84 84 F F F
53202024-02-20T22:43:58.414Z 4 080-099 37 37 37 66 66 66 F F F
53212024-02-20T22:43:58.414Z 5 100-119 47 47 47 88 88 88 F F F
53222024-02-20T22:43:58.414Z 6 120-139 45 45 45 85 85 85 F F F
53232024-02-20T22:43:58.414Z 7 140-159 45 45 45 85 85 85 F F F
53242024-02-20T22:43:58.414Z 8 160-179 47 47 47 88 88 88 F F F
53252024-02-20T22:43:58.414Z 9 180-199 48 47 48 89 88 89 F F F <---
53262024-02-20T22:43:58.414Z 10 200-219 41 41 41 76 76 76 F F F
53272024-02-20T22:43:58.414Z 11 220-239 48 47 48 89 88 89 F F F <---
53282024-02-20T22:43:58.414Z 12 240-259 47 47 47 88 88 88 F F F
53292024-02-20T22:43:58.414Z 13 260-279 47 47 47 88 88 88 F F F
53302024-02-20T22:43:58.414Z 14 280-299 48 47 48 91 88 91 F F F <---
53312024-02-20T22:43:58.414Z 15 300-319 47 47 47 88 88 88 F F F
53322024-02-20T22:43:58.414Z 16 320-339 45 45 45 85 85 85 F F F
53332024-02-20T22:43:58.414Z 17 340-359 43 43 43 81 81 81 F F F
53342024-02-20T22:43:58.414Z 18 360-379 44 44 44 84 84 84 F F F
53352024-02-20T22:43:58.414Z 19 380-399 48 42 48 90 77 90 F F F <---
53362024-02-20T22:43:58.414Z 20 400-419 46 46 46 86 86 86 F F F
53372024-02-20T22:43:58.414Z 21 420-439 41 41 41 76 76 76 T F T <---
53382024-02-20T22:43:58.414Z 22 440-459 47 47 47 88 88 88 T F T <---
53392024-02-20T22:43:58.414Z 23 460-479 48 44 48 91 82 91 F F F <---
53402024-02-20T22:43:58.414Z 24 480-499 48 44 48 91 82 91 F F F <---
53412024-02-20T22:43:58.414Z 25 500-519 48 46 48 91 86 91 F F F <---
53422024-02-20T22:43:58.415Z 26 520-539 48 47 48 90 88 90 F F F <---
53432024-02-20T22:43:58.415Z 27 540-559 47 47 47 88 88 88 T F T <---
53442024-02-20T22:43:58.415Z 28 560-579 48 45 48 89 85 89 F F F <---
53452024-02-20T22:43:58.415Z 29 580-599 48 45 48 89 85 89 F F F <---
53462024-02-20T22:43:58.415ZMax gen: 48, Max flush: 91
53472024-02-20T22:43:58.415ZError: Difference in extent metadata found!
53482024-02-20T22:43:58.415ZA Difference in extent metadata is expected here
53492024-02-20T22:43:58.415ZOn loop 24
53502024-02-20T22:43:58.415Z
53512024-02-20T22:43:58.415ZVerifying data now
53522024-02-20T22:44:02.605ZNo change after verify
53532024-02-20T22:44:02.606ZLoop: 24 Downstairs dump after verify (and repair):
53542024-02-20T22:44:02.631ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53552024-02-20T22:44:02.631Z 0 000-019 46 46 46 87 87 87 F F F
53562024-02-20T22:44:02.632Z 1 020-039 47 47 47 88 88 88 F F F
53572024-02-20T22:44:02.632Z 2 040-059 46 46 46 86 86 86 F F F
53582024-02-20T22:44:02.632Z 3 060-079 44 44 44 84 84 84 F F F
53592024-02-20T22:44:02.632Z 4 080-099 37 37 37 66 66 66 F F F
53602024-02-20T22:44:02.632Z 5 100-119 47 47 47 88 88 88 F F F
53612024-02-20T22:44:02.633Z 6 120-139 45 45 45 85 85 85 F F F
53622024-02-20T22:44:02.633Z 7 140-159 45 45 45 85 85 85 F F F
53632024-02-20T22:44:02.633Z 8 160-179 47 47 47 88 88 88 F F F
53642024-02-20T22:44:02.634Z 9 180-199 48 48 48 89 89 89 F F F
53652024-02-20T22:44:02.634Z 10 200-219 41 41 41 76 76 76 F F F
53662024-02-20T22:44:02.634Z 11 220-239 48 48 48 89 89 89 F F F
53672024-02-20T22:44:02.634Z 12 240-259 47 47 47 88 88 88 F F F
53682024-02-20T22:44:02.634Z 13 260-279 47 47 47 88 88 88 F F F
53692024-02-20T22:44:02.634Z 14 280-299 48 48 48 91 91 91 F F F
53702024-02-20T22:44:02.634Z 15 300-319 47 47 47 88 88 88 F F F
53712024-02-20T22:44:02.634Z 16 320-339 45 45 45 85 85 85 F F F
53722024-02-20T22:44:02.635Z 17 340-359 43 43 43 81 81 81 F F F
53732024-02-20T22:44:02.635Z 18 360-379 44 44 44 84 84 84 F F F
53742024-02-20T22:44:02.635Z 19 380-399 48 48 48 90 90 90 F F F
53752024-02-20T22:44:02.635Z 20 400-419 46 46 46 86 86 86 F F F
53762024-02-20T22:44:02.635Z 21 420-439 49 49 49 92 92 92 F F F
53772024-02-20T22:44:02.635Z 22 440-459 49 49 49 92 92 92 F F F
53782024-02-20T22:44:02.636Z 23 460-479 48 48 48 91 91 91 F F F
53792024-02-20T22:44:02.636Z 24 480-499 48 48 48 91 91 91 F F F
53802024-02-20T22:44:02.636Z 25 500-519 48 48 48 91 91 91 F F F
53812024-02-20T22:44:02.636Z 26 520-539 48 48 48 90 90 90 F F F
53822024-02-20T22:44:02.636Z 27 540-559 49 49 49 92 92 92 F F F
53832024-02-20T22:44:02.636Z 28 560-579 48 48 48 89 89 89 F F F
53842024-02-20T22:44:02.637Z 29 580-599 48 48 48 89 89 89 F F F
53852024-02-20T22:44:02.637ZMax gen: 49, Max flush: 92
53862024-02-20T22:44:02.637Z
53872024-02-20T22:44:02.637ZBegin loop 25
53882024-02-20T22:44:02.638ZDownstairs to restart: 0
53892024-02-20T22:44:02.709Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166357644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335}
53902024-02-20T22:44:02.710Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166690432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335}
53912024-02-20T22:44:02.710Z{"msg":"Upstairs <-> Downstairs Message Version: 5"Crucible runtime is spawned
53922024-02-20T22:44:02.710Z,"v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16674436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335}
53932024-02-20T22:44:02.710Z{"msg":"Crucible df47e1e7-7af2-47b8-b005-71a0a7ba621e has session id: 7fbd48ab-be76-49e0-bd18-58c99f68dd9b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166800588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
53942024-02-20T22:44:02.710Z{"msg":"Upstairs opts: Upstairs UUID: df47e1e7-7af2-47b8-b005-71a0a7ba621e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166846006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
53952024-02-20T22:44:02.710Z{"msg":"Crucible stats registered with UUID: df47e1e7-7af2-47b8-b005-71a0a7ba621e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166887355Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
53962024-02-20T22:44:02.710Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166915864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335}
53972024-02-20T22:44:02.710Z{"msg":"df47e1e7-7af2-47b8-b005-71a0a7ba621e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.166953042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
53982024-02-20T22:44:02.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167000201Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
53992024-02-20T22:44:02.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16703117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54002024-02-20T22:44:02.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167059219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54012024-02-20T22:44:02.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167085648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54022024-02-20T22:44:02.711Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167166525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54032024-02-20T22:44:02.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167214703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54042024-02-20T22:44:02.711Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167254032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54052024-02-20T22:44:02.711Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16729159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54062024-02-20T22:44:02.711Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167327609Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54072024-02-20T22:44:02.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167352658Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54082024-02-20T22:44:02.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167384927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54092024-02-20T22:44:02.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.167421236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54102024-02-20T22:44:02.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.168050053Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54112024-02-20T22:44:02.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.168086152Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54122024-02-20T22:44:02.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.168101551Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54132024-02-20T22:44:02.711Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16812916Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54142024-02-20T22:44:02.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16814331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54152024-02-20T22:44:02.711Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.168156919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54162024-02-20T22:44:02.711Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16869966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54172024-02-20T22:44:02.712Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.168759428Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54182024-02-20T22:44:02.712Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.168853285Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54192024-02-20T22:44:02.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169379726Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54202024-02-20T22:44:02.712Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169425244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54212024-02-20T22:44:02.712Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169460333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54222024-02-20T22:44:02.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169483262Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54232024-02-20T22:44:02.712Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169497371Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54242024-02-20T22:44:02.712Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169517681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54252024-02-20T22:44:02.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16953062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54262024-02-20T22:44:02.712Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169550749Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54272024-02-20T22:44:02.712Z{"msg":"[0]R flush_numbers[0..12]: [87, 88, 86, 84, 66, 88, 85, 85, 88, 89, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169564899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54282024-02-20T22:44:02.712Z{"msg":"[0]R generation[0..12]: [46, 47, 46, 44, 37, 47, 45, 45, 47, 48, 41, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169585678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54292024-02-20T22:44:02.712Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169600938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54302024-02-20T22:44:02.712Z{"msg":"[1]R flush_numbers[0..12]: [87, 88, 86, 84, 66, 88, 85, 85, 88, 89, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169627227Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54312024-02-20T22:44:02.712Z{"msg":"[1]R generation[0..12]: [46, 47, 46, 44, 37, 47, 45, 45, 47, 48, 41, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169644656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
54322024-02-20T22:44:02.712Zdownstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54332024-02-20T22:44:02.712Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169675225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54342024-02-20T22:44:02.712Z{"msg":"[2]R flush_numbers[0..12]: [87, 88, 86, 84, 66, 88, 85, 85, 88, 89, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169708744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54352024-02-20T22:44:02.712Z{"msg":"[2]R generation[0..12]: [46, 47, 46, 44, 37, 47, 45, 45, 47, 48, 41, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169725163Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54362024-02-20T22:44:02.712Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169751882Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54372024-02-20T22:44:02.712Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169776981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54382024-02-20T22:44:02.712Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.16980605Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54392024-02-20T22:44:02.712Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169829059Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54402024-02-20T22:44:02.712Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169861348Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54412024-02-20T22:44:02.712Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169886468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54422024-02-20T22:44:02.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169915626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54432024-02-20T22:44:02.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.169941695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54442024-02-20T22:44:02.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54452024-02-20T22:44:02.713Z2024-02-20T22:44:02.169965645Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54462024-02-20T22:44:02.713Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.170000813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54472024-02-20T22:44:02.713Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.170024172Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54482024-02-20T22:44:02.713Z{"msg":"df47e1e7-7af2-47b8-b005-71a0a7ba621e is now active with session: 7fbd48ab-be76-49e0-bd18-58c99f68dd9b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.170045972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54492024-02-20T22:44:02.713Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.170059761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335,"session_id":"7fbd48ab-be76-49e0-bd18-58c99f68dd9b"}
54502024-02-20T22:44:02.713Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:02.170080741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335}
54512024-02-20T22:44:02.714ZLoading write count information from file "/tmp/test_repair_verify.data"
54522024-02-20T22:44:02.714ZRun Repair workload
54532024-02-20T22:44:02.714Z01/30 Write block 341 len 40960 data: 5 5 5 4 4 4 6 7 6 6
54542024-02-20T22:44:02.714Z02/30 Read block 441 len 8192
54552024-02-20T22:44:02.714Z03/30 Read block 131 len 40960
54562024-02-20T22:44:02.714Z04/30 Read block 187 len 28672
54572024-02-20T22:44:02.714Z05/30 Flush
54582024-02-20T22:44:02.715Z06/30 Write block 128 len 32768 data: 4 4 3 3 3 3 4 6
54592024-02-20T22:44:02.715Z07/30 Write block 65 len 32768 data: 3 3 3 4 5 6 5 5
54602024-02-20T22:44:02.715Z08/30 Read block 62 len 40960
54612024-02-20T22:44:02.715Z09/30 Flush
54622024-02-20T22:44:03.160Z10/30 Write block 477 len 36864 data: 6 6 7 9 8 8 8 7 5
54632024-02-20T22:44:03.160Z11/30 Read block 238 len 28672
54642024-02-20T22:44:03.160Z12/30 Read block 247 len 8192
54652024-02-20T22:44:03.160Z13/30 Write block 460 len 8192 data: 6 6
54662024-02-20T22:44:03.160Z14/30 Write block 561 len 16384 data: 2 2 3 5
54672024-02-20T22:44:03.161Z15/30 Write block 133 len 16384 data: 4 5 7 5
54682024-02-20T22:44:03.161Z16/30 Flush
54692024-02-20T22:44:03.161Z17/30 Write block 264 len 12288 data: 5 5 6
54702024-02-20T22:44:03.161Z18/30 Write block 511 len 24576 data: 8 4 4 4 5 4
54712024-02-20T22:44:03.161Z19/30 Write block 497 len 8192 data: 9 8
54722024-02-20T22:44:03.161Z20/30 Read block 21 len 4096
54732024-02-20T22:44:03.161Z21/30 Write block 283 len 24576 data: 7 7 6 7 7 6
54742024-02-20T22:44:03.161Z22/30 Read block 242 len 32768
54752024-02-20T22:44:03.161Z23/30 Flush
54762024-02-20T22:44:03.161Z24/30 Write block 419 len 24576 data: 5 5 4 3 3 3
54772024-02-20T22:44:03.161Z25/30 Read block 460 len 12288
54782024-02-20T22:44:03.161Z26/30 Write block 452 len 4096 data: 7
54792024-02-20T22:44:03.161Z27/30 Write block 156 len 16384 data: 4 5 5 5
54802024-02-20T22:44:03.161Z28/30 Read block 417 len 40960
54812024-02-20T22:44:03.161Z29/30 Write block 358 len 40960 data: 5 7 8 8 8 11 9 12 10 6
54822024-02-20T22:44:03.161Z30/30 Read block 545 len 40960
54832024-02-20T22:44:03.161Z----------------------------------------------------------------
54842024-02-20T22:44:03.161Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:25
54852024-02-20T22:44:03.162ZStates: Active Active Active
54862024-02-20T22:44:03.162ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54872024-02-20T22:44:03.162Z 6 Acked 1005 Write 8 Done Done Done false
54882024-02-20T22:44:03.162Z 7 Acked 1006 Write 8 Done Done Done false
54892024-02-20T22:44:03.162Z 8 Acked 1007 Read 10 Sent Done Done false
54902024-02-20T22:44:03.162Z 9 Acked 1008 Flush 0 Sent Done Done false
54912024-02-20T22:44:03.162Z 10 Acked 1009 Write 9 Sent Done Done false
54922024-02-20T22:44:03.162Z 11 Acked 1010 Read 7 Sent Done Done false
54932024-02-20T22:44:03.162Z 12 Acked 1011 Read 2 Sent Done Done false
54942024-02-20T22:44:03.162Z 13 Acked 1012 Write 2 Sent Done Done false
54952024-02-20T22:44:03.162Z 14 Acked 1013 Write 4 Sent Done Done false
54962024-02-20T22:44:03.162Z 15 Acked 1014 Write 4 Sent Done Done false
54972024-02-20T22:44:03.162Z 16 Acked 1015 Flush 0 Sent Done Done false
54982024-02-20T22:44:03.162Z 17 Acked 1016 Write 3 Sent Done Done false
54992024-02-20T22:44:03.162Z 18 Acked 1017 Write 6 Sent Done Done false
55002024-02-20T22:44:03.162Z 19 Acked 1018 Write 2 Sent Done Done false
55012024-02-20T22:44:03.162Z 20 Acked 1019 Read 1 Sent Done Done false
55022024-02-20T22:44:03.162Z 21 Acked 1020 Write 6 Sent Done Done false
55032024-02-20T22:44:03.163Z 22 Acked 1021 Read 8 Sent Done Done false
55042024-02-20T22:44:03.163Z 23 Acked 1022 Flush 0 Sent Done Done false
55052024-02-20T22:44:03.163Z 24 Acked 1023 Write 6 Sent Done Done false
55062024-02-20T22:44:03.163Z 25 Acked 1024 Read 3 Sent Done Done false
55072024-02-20T22:44:03.163Z 26 Acked 1025 Write 1 Sent Done Done false
55082024-02-20T22:44:03.163Z 27 Acked 1026 Write 4 Sent Done Done false
55092024-02-20T22:44:03.163Z 28 Acked 1027 Read 10 Sent Done Done false
55102024-02-20T22:44:03.163Z 29 Acked 1028 Write 10 Sent Done Done false
55112024-02-20T22:44:03.163Z 30 Acked 1029 Read 10 Sent Done Sent false
55122024-02-20T22:44:03.163Z STATES DS:0 DS:1 DS:2 TOTAL
55132024-02-20T22:44:03.163Z New 0 0 0 0
55142024-02-20T22:44:03.163Z Sent 23 0 1 24
55152024-02-20T22:44:03.164Z Done 2 25 24 51
55162024-02-20T22:44:03.164Z Skipped 0 0 0 0
55172024-02-20T22:44:03.164Z Error 0 0 0 0
55182024-02-20T22:44:03.164ZLast Flush: 1004 1022 1022
55192024-02-20T22:44:03.164ZDownstairs last five completed: 1004 1003 1002 1001 1000
55202024-02-20T22:44:03.164ZUpstairs last five completed: 30 29 28 27 26
55212024-02-20T22:44:03.164Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:02.204322385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1335}
55222024-02-20T22:44:03.164ZWrote out file "/tmp/test_repair_verify.data"
55232024-02-20T22:44:03.164Z
55242024-02-20T22:44:03.164ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55252024-02-20T22:44:03.164ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55262024-02-20T22:44:03.164Z 0 000-019 46 46 46 87 87 87 F F F
55272024-02-20T22:44:03.164Z 1 020-039 47 47 47 88 88 88 F F F
55282024-02-20T22:44:03.165Z 2 040-059 46 46 46 86 86 86 F F F
55292024-02-20T22:44:03.165Z 3 060-079 44 50 50 84 94 94 T F F <---
55302024-02-20T22:44:03.165Z 4 080-099 37 37 37 66 66 66 F F F
55312024-02-20T22:44:03.165Z 5 100-119 47 47 47 88 88 88 F F F
55322024-02-20T22:44:03.165Z 6 120-139 45 50 50 85 95 95 T F F <---
55332024-02-20T22:44:03.165Z 7 140-159 45 45 45 85 85 85 F T T <---
55342024-02-20T22:44:03.165Z 8 160-179 47 47 47 88 88 88 F F F
55352024-02-20T22:44:03.165Z 9 180-199 48 48 48 89 89 89 F F F
55362024-02-20T22:44:03.165Z 10 200-219 41 41 41 76 76 76 F F F
55372024-02-20T22:44:03.165Z 11 220-239 48 48 48 89 89 89 F F F
55382024-02-20T22:44:03.165Z 12 240-259 47 47 47 88 88 88 F F F
55392024-02-20T22:44:03.165Z 13 260-279 47 50 50 88 96 96 F F F <---
55402024-02-20T22:44:03.166Z 14 280-299 48 50 50 91 96 96 F F F <---
55412024-02-20T22:44:03.166Z 15 300-319 47 47 47 88 88 88 F F F
55422024-02-20T22:44:03.166Z 16 320-339 45 45 45 85 85 85 F F F
55432024-02-20T22:44:03.166Z 17 340-359 50 50 50 93 93 93 F T T <---
55442024-02-20T22:44:03.166Z 18 360-379 44 44 44 84 84 84 F T T <---
55452024-02-20T22:44:03.166Z 19 380-399 48 48 48 90 90 90 F F F
55462024-02-20T22:44:03.166Z 20 400-419 46 46 46 86 86 86 F T T <---
55472024-02-20T22:44:03.166Z 21 420-439 49 49 49 92 92 92 F T T <---
55482024-02-20T22:44:03.166Z 22 440-459 49 49 49 92 92 92 F T T <---
55492024-02-20T22:44:03.166Z 23 460-479 48 50 50 91 95 95 F F F <---
55502024-02-20T22:44:03.167Z 24 480-499 48 50 50 91 96 96 F F F <---
55512024-02-20T22:44:03.167Z 25 500-519 48 50 50 91 96 96 F F F <---
55522024-02-20T22:44:03.167Z 26 520-539 48 48 48 90 90 90 F F F
55532024-02-20T22:44:03.167Z 27 540-559 49 49 49 92 92 92 F F F
55542024-02-20T22:44:03.167Z 28 560-579 48 50 50 89 95 95 F F F <---
55552024-02-20T22:44:03.167Z 29 580-599 48 48 48 89 89 89 F F F
55562024-02-20T22:44:03.167ZMax gen: 50, Max flush: 96
55572024-02-20T22:44:03.167ZError: Difference in extent metadata found!
55582024-02-20T22:44:03.167ZA Difference in extent metadata is expected here
55592024-02-20T22:44:03.167ZOn loop 25
55602024-02-20T22:44:03.168Z
55612024-02-20T22:44:03.168ZVerifying data now
55622024-02-20T22:44:07.385ZNo change after verify
55632024-02-20T22:44:07.385ZLoop: 25 Downstairs dump after verify (and repair):
55642024-02-20T22:44:07.412ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55652024-02-20T22:44:07.412Z 0 000-019 46 46 46 87 87 87 F F F
55662024-02-20T22:44:07.412Z 1 020-039 47 47 47 88 88 88 F F F
55672024-02-20T22:44:07.412Z 2 040-059 46 46 46 86 86 86 F F F
55682024-02-20T22:44:07.412Z 3 060-079 50 50 50 94 94 94 F F F
55692024-02-20T22:44:07.412Z 4 080-099 37 37 37 66 66 66 F F F
55702024-02-20T22:44:07.412Z 5 100-119 47 47 47 88 88 88 F F F
55712024-02-20T22:44:07.412Z 6 120-139 50 50 50 95 95 95 F F F
55722024-02-20T22:44:07.413Z 7 140-159 51 51 51 97 97 97 F F F
55732024-02-20T22:44:07.413Z 8 160-179 47 47 47 88 88 88 F F F
55742024-02-20T22:44:07.413Z 9 180-199 48 48 48 89 89 89 F F F
55752024-02-20T22:44:07.413Z 10 200-219 41 41 41 76 76 76 F F F
55762024-02-20T22:44:07.413Z 11 220-239 48 48 48 89 89 89 F F F
55772024-02-20T22:44:07.413Z 12 240-259 47 47 47 88 88 88 F F F
55782024-02-20T22:44:07.413Z 13 260-279 50 50 50 96 96 96 F F F
55792024-02-20T22:44:07.413Z 14 280-299 50 50 50 96 96 96 F F F
55802024-02-20T22:44:07.413Z 15 300-319 47 47 47 88 88 88 F F F
55812024-02-20T22:44:07.413Z 16 320-339 45 45 45 85 85 85 F F F
55822024-02-20T22:44:07.413Z 17 340-359 51 51 51 97 97 97 F F F
55832024-02-20T22:44:07.413Z 18 360-379 51 51 51 97 97 97 F F F
55842024-02-20T22:44:07.413Z 19 380-399 48 48 48 90 90 90 F F F
55852024-02-20T22:44:07.413Z 20 400-419 51 51 51 97 97 97 F F F
55862024-02-20T22:44:07.413Z 21 420-439 51 51 51 97 97 97 F F F
55872024-02-20T22:44:07.413Z 22 440-459 51 51 51 97 97 97 F F F
55882024-02-20T22:44:07.414Z 23 460-479 50 50 50 95 95 95 F F F
55892024-02-20T22:44:07.414Z 24 480-499 50 50 50 96 96 96 F F F
55902024-02-20T22:44:07.414Z 25 500-519 50 50 50 96 96 96 F F F
55912024-02-20T22:44:07.414Z 26 520-539 48 48 48 90 90 90 F F F
55922024-02-20T22:44:07.414Z 27 540-559 49 49 49 92 92 92 F F F
55932024-02-20T22:44:07.414Z 28 560-579 50 50 50 95 95 95 F F F
55942024-02-20T22:44:07.414Z 29 580-599 48 48 48 89 89 89 F F F
55952024-02-20T22:44:07.414ZMax gen: 51, Max flush: 97
55962024-02-20T22:44:07.414Z
55972024-02-20T22:44:07.414ZBegin loop 26
55982024-02-20T22:44:07.414ZDownstairs to restart: 1
55992024-02-20T22:44:07.421Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.953437819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343}
56002024-02-20T22:44:07.434Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.953775596Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343}
56012024-02-20T22:44:07.434ZCrucible runtime is spawned
56022024-02-20T22:44:07.434Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.953830284Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343}
56032024-02-20T22:44:07.434Z{"msg":"Crucible 5de4722e-5111-470f-b9d4-a01eb15e6781 has session id: 8af666a7-43df-42d3-acef-4e632c4c20ba","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.953861643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56042024-02-20T22:44:07.434Z{"msg":"Upstairs opts: Upstairs UUID: 5de4722e-5111-470f-b9d4-a01eb15e6781, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.953921991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56052024-02-20T22:44:07.434Z{"msg":"Crucible stats registered with UUID: 5de4722e-5111-470f-b9d4-a01eb15e6781","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95395878Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56062024-02-20T22:44:07.434Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.953988039Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343}
56072024-02-20T22:44:07.434Z{"msg":"5de4722e-5111-470f-b9d4-a01eb15e6781 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954166222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56082024-02-20T22:44:07.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95423248Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56092024-02-20T22:44:07.434Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954271539Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56102024-02-20T22:44:07.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954303437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56112024-02-20T22:44:07.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954338426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56122024-02-20T22:44:07.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954364735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56132024-02-20T22:44:07.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954390014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56142024-02-20T22:44:07.435Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954432113Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56152024-02-20T22:44:07.435Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954469382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56162024-02-20T22:44:07.435Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95450235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56172024-02-20T22:44:07.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954534009Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56182024-02-20T22:44:07.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954592017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56192024-02-20T22:44:07.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.954649445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56202024-02-20T22:44:07.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.955312981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56212024-02-20T22:44:07.435Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95534193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56222024-02-20T22:44:07.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95535777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56232024-02-20T22:44:07.435Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.955371809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56242024-02-20T22:44:07.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.955404818Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56252024-02-20T22:44:07.435Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.955419517Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56262024-02-20T22:44:07.435Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.955959718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56272024-02-20T22:44:07.435Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956005156Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56282024-02-20T22:44:07.435Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95617413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56292024-02-20T22:44:07.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95646336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56302024-02-20T22:44:07.436Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956518528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56312024-02-20T22:44:07.436Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956546427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56322024-02-20T22:44:07.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956569916Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56332024-02-20T22:44:07.436Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956596155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56342024-02-20T22:44:07.436Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956621365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56352024-02-20T22:44:07.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956661193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56362024-02-20T22:44:07.436Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956682062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56372024-02-20T22:44:07.436Z{"msg":"[0]R flush_numbers[0..12]: [87, 88, 86, 94, 66, 88, 95, 97, 88, 89, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956712541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56382024-02-20T22:44:07.436Z{"msg":"[0]R generation[0..12]: [46, 47, 46, 50, 37, 47, 50, 51, 47, 48, 41, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95674529Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56392024-02-20T22:44:07.436Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956766259Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56402024-02-20T22:44:07.436Z{"msg":"[1]R flush_numbers[0..12]: [87, 88, 86, 94, 66, 88, 95, 97, 88, 89, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956795698Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56412024-02-20T22:44:07.436Z{"msg":"[1]R generation[0..12]: [46, 47, 46, 50, 37, 47, 50, 51, 47, 48, 41, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956815418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56422024-02-20T22:44:07.436Z{"msg":"Wait for a query_work_queue command to finish before sending IO
56432024-02-20T22:44:07.436Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956849196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56442024-02-20T22:44:07.436Z{"msg":"[2]R flush_numbers[0..12]: [87, 88, 86, 94, 66, 88, 95, 97, 88, 89, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956881945Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56452024-02-20T22:44:07.436Z{"msg":"[2]R generation[0..12]: [46, 47, 46, 50, 37, 47, 50, 51, 47, 48, 41, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956914734Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56462024-02-20T22:44:07.436Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956947993Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56472024-02-20T22:44:07.436Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956967362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56482024-02-20T22:44:07.436Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.956997061Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56492024-02-20T22:44:07.436Z{"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.9570146Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56502024-02-20T22:44:07.436Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.95704333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56512024-02-20T22:44:07.437Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957064269Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56522024-02-20T22:44:07.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957090748Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56532024-02-20T22:44:07.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957110247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56542024-02-20T22:44:07.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957138616Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56552024-02-20T22:44:07.437Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957157625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56562024-02-20T22:44:07.437Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957185244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56572024-02-20T22:44:07.437Z{"msg":"5de4722e-5111-470f-b9d4-a01eb15e6781 is now active with session: 8af666a7-43df-42d3-acef-4e632c4c20ba","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957203674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56582024-02-20T22:44:07.437Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957230643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343,"session_id":"8af666a7-43df-42d3-acef-4e632c4c20ba"}
56592024-02-20T22:44:07.437Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:06.957247392Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343}
56602024-02-20T22:44:07.437ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56612024-02-20T22:44:07.437ZLoading write count information from file "/tmp/test_repair_verify.data"
56622024-02-20T22:44:07.437ZRun Repair workload
56632024-02-20T22:44:07.442Z01/30 Write block 5 len 28672 data: 5 4 3 3 3 4 3
56642024-02-20T22:44:07.442Z02/30 Write block 579 len 8192 data: 6 6
56652024-02-20T22:44:07.442Z03/30 Read block 548 len 36864
56662024-02-20T22:44:07.442Z04/30 Read block 590 len 32768
56672024-02-20T22:44:07.442Z05/30 Write block 115 len 4096 data: 5
56682024-02-20T22:44:07.442Z06/30 Read block 551 len 8192
56692024-02-20T22:44:07.442Z07/30 Write block 38 len 28672 data: 6 7 4 4 5 4 4
56702024-02-20T22:44:07.442Z08/30 Read block 35 len 4096
56712024-02-20T22:44:07.442Z09/30 Write block 545 len 20480 data: 4 3 5 5 6
56722024-02-20T22:44:07.442Z10/30 Read block 567 len 24576
56732024-02-20T22:44:07.442Z11/30 Read block 424 len 28672
56742024-02-20T22:44:07.442Z12/30 Write block 120 len 8192 data: 5 4
56752024-02-20T22:44:07.442Z13/30 Read block 274 len 20480
56762024-02-20T22:44:07.442Z14/30 Read block 7 len 16384
56772024-02-20T22:44:07.443Z15/30 Read block 243 len 40960
56782024-02-20T22:44:07.443Z16/30 Read block 508 len 40960
56792024-02-20T22:44:07.443Z17/30 Write block 494 len 32768 data: 6 6 8 10 9 7 6 5
56802024-02-20T22:44:07.443Z18/30 Write block 436 len 16384 data: 2 3 4 6
56812024-02-20T22:44:07.443Z19/30 Flush
56822024-02-20T22:44:07.447Z20/30 Flush
56832024-02-20T22:44:07.447Z21/30 Write block 529 len 8192 data: 6 7
56842024-02-20T22:44:07.447Z22/30 Read block 27 len 8192
56852024-02-20T22:44:07.451Z23/30 Read block 298 len 36864
56862024-02-20T22:44:07.451Z24/30 Write block 573 len 4096 data: 3
56872024-02-20T22:44:07.451Z25/30 Read block 293 len 24576
56882024-02-20T22:44:07.452Z26/30 Read block 489 len 24576
56892024-02-20T22:44:07.452Z27/30 Write block 233 len 8192 data: 9 9
56902024-02-20T22:44:07.452Z28/30 Read block 264 len 28672
56912024-02-20T22:44:07.458Z29/30 Write block 440 len 36864 data: 7 7 8 8 7 8 8 7 6
56922024-02-20T22:44:07.458Z30/30 Write block 472 len 40960 data: 3 3 4 5 5 7 7 8 10 9
56932024-02-20T22:44:07.458Z----------------------------------------------------------------
56942024-02-20T22:44:07.458Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56952024-02-20T22:44:07.458ZStates: Active Active Active
56962024-02-20T22:44:07.458ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56972024-02-20T22:44:07.458Z 1 Acked 1000 Write 7 Done Done Done false
56982024-02-20T22:44:07.458Z 2 Acked 1001 Write 2 Done Done Done false
56992024-02-20T22:44:07.458Z 3 Acked 1002 Read 9 Done Done Done false
57002024-02-20T22:44:07.458Z 4 Acked 1003 Read 8 Done Sent Done false
57012024-02-20T22:44:07.458Z 5 Acked 1004 Write 1 Done Sent Done false
57022024-02-20T22:44:07.458Z 6 Acked 1005 Read 2 Done Sent Done false
57032024-02-20T22:44:07.458Z 7 Acked 1006 Write 7 Done Sent Done false
57042024-02-20T22:44:07.459Z 8 Acked 1007 Read 1 Done Sent Done false
57052024-02-20T22:44:07.459Z 9 Acked 1008 Write 5 Done Sent Done false
57062024-02-20T22:44:07.459Z 10 Acked 1009 Read 6 Done Sent Done false
57072024-02-20T22:44:07.459Z 11 Acked 1010 Read 7 Done Sent Done false
57082024-02-20T22:44:07.459Z 12 Acked 1011 Write 2 Done Sent Done false
57092024-02-20T22:44:07.459Z 13 Acked 1012 Read 5 Done Sent Done false
57102024-02-20T22:44:07.459Z 14 Acked 1013 Read 4 Done Sent Done false
57112024-02-20T22:44:07.459Z 15 Acked 1014 Read 10 Done Sent Done false
57122024-02-20T22:44:07.459Z 16 Acked 1015 Read 10 Done Sent Done false
57132024-02-20T22:44:07.459Z 17 Acked 1016 Write 8 Done Sent Done false
57142024-02-20T22:44:07.459Z 18 Acked 1017 Write 4 Done Sent Done false
57152024-02-20T22:44:07.459Z 19 Acked 1018 Flush 0 Done Sent Done false
57162024-02-20T22:44:07.459Z 20 Acked 1019 Flush 0 Done Sent Done false
57172024-02-20T22:44:07.459Z 21 Acked 1020 Write 2 Done Sent Done false
57182024-02-20T22:44:07.459Z 22 Acked 1021 Read 2 Done Sent Done false
57192024-02-20T22:44:07.459Z 23 Acked 1022 Read 9 Done Sent Done false
57202024-02-20T22:44:07.459Z 24 Acked 1023 Write 1 Done Sent Done false
57212024-02-20T22:44:07.460Z 25 Acked 1024 Read 6 Done Sent Done false
57222024-02-20T22:44:07.460Z 26 Acked 1025 Read 6 Done Sent Done false
57232024-02-20T22:44:07.460Z 27 Acked 1026 Write 2 Done Sent Done false
57242024-02-20T22:44:07.460Z 28 Acked 1027 Read 7 Done Sent Done false
57252024-02-20T22:44:07.460Z 29 Acked 1028 Write 9 Done Sent Sent false
57262024-02-20T22:44:07.460Z 30 Acked 1029 Write 10 Sent Sent Sent false
57272024-02-20T22:44:07.460Z STATES DS:0 DS:1 DS:2 TOTAL
57282024-02-20T22:44:07.460Z New 0 0 0 0
57292024-02-20T22:44:07.460Z Sent 1 27 2 30
57302024-02-20T22:44:07.460Z Done 29 3 28 60
57312024-02-20T22:44:07.460Z Skipped 0 0 0 0
57322024-02-20T22:44:07.460Z Error 0 0 0 0
57332024-02-20T22:44:07.460ZLast Flush: 1019 0 1019
57342024-02-20T22:44:07.460ZDownstairs last five completed:
57352024-02-20T22:44:07.460ZUpstairs last five completed: 30 29 28 27 26
57362024-02-20T22:44:07.460Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:06.986783845Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1343}
57372024-02-20T22:44:07.460ZWrote out file "/tmp/test_repair_verify.data"
57382024-02-20T22:44:07.466Z
57392024-02-20T22:44:07.466ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57402024-02-20T22:44:07.485ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57412024-02-20T22:44:07.485Z 0 000-019 52 46 52 98 87 98 F T F <---
57422024-02-20T22:44:07.485Z 1 020-039 52 47 52 98 88 98 F F F <---
57432024-02-20T22:44:07.485Z 2 040-059 52 46 52 98 86 98 F F F <---
57442024-02-20T22:44:07.485Z 3 060-079 50 50 50 94 94 94 F F F
57452024-02-20T22:44:07.485Z 4 080-099 37 37 37 66 66 66 F F F
57462024-02-20T22:44:07.485Z 5 100-119 52 47 52 98 88 98 F F F <---
57472024-02-20T22:44:07.486Z 6 120-139 52 50 52 98 95 98 F F F <---
57482024-02-20T22:44:07.486Z 7 140-159 51 51 51 97 97 97 F F F
57492024-02-20T22:44:07.486Z 8 160-179 47 47 47 88 88 88 F F F
57502024-02-20T22:44:07.486Z 9 180-199 48 48 48 89 89 89 F F F
57512024-02-20T22:44:07.486Z 10 200-219 41 41 41 76 76 76 F F F
57522024-02-20T22:44:07.492Z 11 220-239 48 48 48 89 89 89 T F T <---
57532024-02-20T22:44:07.493Z 12 240-259 47 47 47 88 88 88 F F F
57542024-02-20T22:44:07.493Z 13 260-279 50 50 50 96 96 96 F F F
57552024-02-20T22:44:07.493Z 14 280-299 50 50 50 96 96 96 F F F
57562024-02-20T22:44:07.493Z 15 300-319 47 47 47 88 88 88 F F F
57572024-02-20T22:44:07.493Z 16 320-339 45 45 45 85 85 85 F F F
57582024-02-20T22:44:07.493Z 17 340-359 51 51 51 97 97 97 F F F
57592024-02-20T22:44:07.493Z 18 360-379 51 51 51 97 97 97 F F F
57602024-02-20T22:44:07.493Z 19 380-399 48 48 48 90 90 90 F F F
57612024-02-20T22:44:07.493Z 20 400-419 51 51 51 97 97 97 F F F
57622024-02-20T22:44:07.493Z 21 420-439 52 51 52 98 97 98 F F F <---
57632024-02-20T22:44:07.493Z 22 440-459 51 51 51 97 97 97 T F T <---
57642024-02-20T22:44:07.493Z 23 460-479 50 50 50 95 95 95 T F T <---
57652024-02-20T22:44:07.493Z 24 480-499 52 50 52 98 96 98 T F T <---
57662024-02-20T22:44:07.493Z 25 500-519 52 50 52 98 96 98 F F F <---
57672024-02-20T22:44:07.493Z 26 520-539 48 48 48 90 90 90 T F T <---
57682024-02-20T22:44:07.494Z 27 540-559 52 49 52 98 92 98 F F F <---
57692024-02-20T22:44:07.494Z 28 560-579 52 50 52 98 95 98 T T T <---
57702024-02-20T22:44:07.494Z 29 580-599 52 48 52 98 89 98 F T F <---
57712024-02-20T22:44:07.494ZMax gen: 52, Max flush: 98
57722024-02-20T22:44:07.494ZError: Difference in extent metadata found!
57732024-02-20T22:44:07.494ZA Difference in extent metadata is expected here
57742024-02-20T22:44:07.494ZOn loop 26
57752024-02-20T22:44:07.494Z
57762024-02-20T22:44:07.494ZVerifying data now
57772024-02-20T22:44:12.202ZNo change after verify
57782024-02-20T22:44:12.202ZLoop: 26 Downstairs dump after verify (and repair):
57792024-02-20T22:44:12.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57802024-02-20T22:44:12.217Z 0 000-019 52 52 52 98 98 98 F F F
57812024-02-20T22:44:12.217Z 1 020-039 52 52 52 98 98 98 F F F
57822024-02-20T22:44:12.217Z 2 040-059 52 52 52 98 98 98 F F F
57832024-02-20T22:44:12.217Z 3 060-079 50 50 50 94 94 94 F F F
57842024-02-20T22:44:12.217Z 4 080-099 37 37 37 66 66 66 F F F
57852024-02-20T22:44:12.217Z 5 100-119 52 52 52 98 98 98 F F F
57862024-02-20T22:44:12.217Z 6 120-139 52 52 52 98 98 98 F F F
57872024-02-20T22:44:12.217Z 7 140-159 51 51 51 97 97 97 F F F
57882024-02-20T22:44:12.217Z 8 160-179 47 47 47 88 88 88 F F F
57892024-02-20T22:44:12.222Z 9 180-199 48 48 48 89 89 89 F F F
57902024-02-20T22:44:12.222Z 10 200-219 41 41 41 76 76 76 F F F
57912024-02-20T22:44:12.222Z 11 220-239 53 53 53 99 99 99 F F F
57922024-02-20T22:44:12.223Z 12 240-259 47 47 47 88 88 88 F F F
57932024-02-20T22:44:12.223Z 13 260-279 50 50 50 96 96 96 F F F
57942024-02-20T22:44:12.223Z 14 280-299 50 50 50 96 96 96 F F F
57952024-02-20T22:44:12.223Z 15 300-319 47 47 47 88 88 88 F F F
57962024-02-20T22:44:12.223Z 16 320-339 45 45 45 85 85 85 F F F
57972024-02-20T22:44:12.223Z 17 340-359 51 51 51 97 97 97 F F F
57982024-02-20T22:44:12.223Z 18 360-379 51 51 51 97 97 97 F F F
57992024-02-20T22:44:12.223Z 19 380-399 48 48 48 90 90 90 F F F
58002024-02-20T22:44:12.223Z 20 400-419 51 51 51 97 97 97 F F F
58012024-02-20T22:44:12.223Z 21 420-439 52 52 52 98 98 98 F F F
58022024-02-20T22:44:12.223Z 22 440-459 53 53 53 99 99 99 F F F
58032024-02-20T22:44:12.223Z 23 460-479 53 53 53 99 99 99 F F F
58042024-02-20T22:44:12.224Z 24 480-499 53 53 53 99 99 99 F F F
58052024-02-20T22:44:12.224Z 25 500-519 52 52 52 98 98 98 F F F
58062024-02-20T22:44:12.224Z 26 520-539 53 53 53 99 99 99 F F F
58072024-02-20T22:44:12.224Z 27 540-559 52 52 52 98 98 98 F F F
58082024-02-20T22:44:12.224Z 28 560-579 53 53 53 99 99 99 F F F
58092024-02-20T22:44:12.224Z 29 580-599 52 52 52 98 98 98 F F F
58102024-02-20T22:44:12.224ZMax gen: 53, Max flush: 99
58112024-02-20T22:44:12.224Z
58122024-02-20T22:44:12.224ZBegin loop 27
58132024-02-20T22:44:12.224ZDownstairs to restart: 2
58142024-02-20T22:44:12.241Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.771977548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351}
58152024-02-20T22:44:12.241Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772263937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351}
58162024-02-20T22:44:12.241Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772307126Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351}
58172024-02-20T22:44:12.241ZCrucible runtime is spawned
58182024-02-20T22:44:12.250Z{"msg":"Crucible 0dc5baa1-74c1-4463-9172-7fe2d04a2801 has session id: b4cb0607-81ae-4cd5-b770-b8d6d165aa59","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772377643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58192024-02-20T22:44:12.250Z{"msg":"Upstairs opts: Upstairs UUID: 0dc5baa1-74c1-4463-9172-7fe2d04a2801, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772446101Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58202024-02-20T22:44:12.251Z{"msg":"Crucible stats registered with UUID: 0dc5baa1-74c1-4463-9172-7fe2d04a2801","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772524458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58212024-02-20T22:44:12.251Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772567626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351}
58222024-02-20T22:44:12.251Z{"msg":"0dc5baa1-74c1-4463-9172-7fe2d04a2801 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772606255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58232024-02-20T22:44:12.251Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772647374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58242024-02-20T22:44:12.251Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772695622Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58252024-02-20T22:44:12.251Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.77274262Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58262024-02-20T22:44:12.251Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772778879Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58272024-02-20T22:44:12.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772819117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58282024-02-20T22:44:12.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772852606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58292024-02-20T22:44:12.252Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772884515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58302024-02-20T22:44:12.252Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772918934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58312024-02-20T22:44:12.252Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.772966752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58322024-02-20T22:44:12.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773000191Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58332024-02-20T22:44:12.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.77303405Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58342024-02-20T22:44:12.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773064049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58352024-02-20T22:44:12.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773661677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58362024-02-20T22:44:12.253Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773699786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58372024-02-20T22:44:12.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773727705Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58382024-02-20T22:44:12.253Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773743035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58392024-02-20T22:44:12.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773773274Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58402024-02-20T22:44:12.253Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.773810062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58412024-02-20T22:44:12.253Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774300855Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58422024-02-20T22:44:12.253Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774341803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58432024-02-20T22:44:12.253Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774371322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58442024-02-20T22:44:12.254Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774821496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58452024-02-20T22:44:12.254Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774868314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58462024-02-20T22:44:12.254Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774896283Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58472024-02-20T22:44:12.254Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774927142Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58482024-02-20T22:44:12.254Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.774951961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58492024-02-20T22:44:12.254Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.77497639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58502024-02-20T22:44:12.254Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775065607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58512024-02-20T22:44:12.254Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775102736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58522024-02-20T22:44:12.255Z{"msg":"[0]R flush_numbers[0..12]: [98, 98, 98, 94, 66, 98, 98, 97, 88, 89, 76, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775120235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58532024-02-20T22:44:12.255Z{"msg":"[0]R generation[0..12]: [52, 52, 52, 50, 37, 52, 52, 51, 47, 48, 41, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775143184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58542024-02-20T22:44:12.255Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775158574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58552024-02-20T22:44:12.255Z{"msg":"[1]R flush_numbers[0..12]: [98, 98, 98, 94, 66, 98, 98, 97, 88, 89, 76, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775179693Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58562024-02-20T22:44:12.255Z{"msg":"[1]R generation[0..12]: [52, 52, 52, 50, 37, 52, 52, 51, 47, 48, 41, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775194832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58572024-02-20T22:44:12.255Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775215512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58582024-02-20T22:44:12.255Z{"msg":"[2]R flush_numbers[0..12]: [98, 98, 98, 94, 66, 98, 98, 97, 88, 89, 76, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775229841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58592024-02-20T22:44:12.255Z{"msg":"[2]R generation[0..12]: [52, 52, 52, 50, 37, 52, 52, 51, 47, 48, 41, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775244031Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58602024-02-20T22:44:12.256ZWait for a query_work_queue command to finish before sending IO
58612024-02-20T22:44:12.256Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.77526923Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58622024-02-20T22:44:12.256Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775283169Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58632024-02-20T22:44:12.256Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775304409Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58642024-02-20T22:44:12.256Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775318278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58652024-02-20T22:44:12.256Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775339017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58662024-02-20T22:44:12.256Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775353527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58672024-02-20T22:44:12.256Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775365806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58682024-02-20T22:44:12.257Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775385936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58692024-02-20T22:44:12.257Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775399425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58702024-02-20T22:44:12.257Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775412875Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58712024-02-20T22:44:12.257Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775435004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58722024-02-20T22:44:12.257Z{"msg":"0dc5baa1-74c1-4463-9172-7fe2d04a2801 is now active with session: b4cb0607-81ae-4cd5-b770-b8d6d165aa59","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775448443Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58732024-02-20T22:44:12.257Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775460403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351,"session_id":"b4cb0607-81ae-4cd5-b770-b8d6d165aa59"}
58742024-02-20T22:44:12.257Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:11.775479302Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351}
58752024-02-20T22:44:12.257ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58762024-02-20T22:44:12.257ZLoading write count information from file "/tmp/test_repair_verify.data"
58772024-02-20T22:44:12.257ZRun Repair workload
58782024-02-20T22:44:12.258Z01/30 Flush
58792024-02-20T22:44:12.258Z02/30 Write block 456 len 16384 data: 8 9 9 7
58802024-02-20T22:44:12.268Z03/30 Read block 410 len 40960
58812024-02-20T22:44:12.268Z04/30 Write block 29 len 24576 data: 14 13 13 12 12 9
58822024-02-20T22:44:12.268Z05/30 Read block 595 len 16384
58832024-02-20T22:44:12.268Z06/30 Read block 391 len 12288
58842024-02-20T22:44:12.269Z07/30 Write block 590 len 36864 data: 8 7 6 6 7 7 6 6 4
58852024-02-20T22:44:12.269Z08/30 Read block 96 len 16384
58862024-02-20T22:44:12.269Z09/30 Read block 517 len 36864
58872024-02-20T22:44:12.269Z10/30 Write block 317 len 16384 data: 7 6 7 8
58882024-02-20T22:44:12.269Z11/30 Read block 525 len 24576
58892024-02-20T22:44:12.269Z12/30 Write block 131 len 28672 data: 4 4 5 6 8 6 5
58902024-02-20T22:44:12.269Z13/30 Write block 245 len 40960 data: 2 2 4 3 3 3 2 2 4 3
58912024-02-20T22:44:12.269Z14/30 Write block 197 len 20480 data: 5 4 5 4 4
58922024-02-20T22:44:12.269Z15/30 Read block 202 len 32768
58932024-02-20T22:44:12.269Z16/30 Read block 378 len 24576
58942024-02-20T22:44:12.270Z17/30 Read block 328 len 4096
58952024-02-20T22:44:12.270Z18/30 Read block 167 len 16384
58962024-02-20T22:44:12.270Z19/30 Write block 468 len 28672 data: 8 9 6 4 4 4 5
58972024-02-20T22:44:12.270Z20/30 Read block 19 len 8192
58982024-02-20T22:44:12.276Z21/30 Write block 167 len 16384 data: 4 5 5 6
58992024-02-20T22:44:12.276Z22/30 Write block 332 len 24576 data: 7 8 7 6 6 6
59002024-02-20T22:44:12.276Z23/30 Read block 212 len 20480
59012024-02-20T22:44:12.276Z24/30 Write block 54 len 24576 data: 5 5 4 3 4 5
59022024-02-20T22:44:12.276Z25/30 Read block 120 len 12288
59032024-02-20T22:44:12.276Z26/30 Read block 486 len 28672
59042024-02-20T22:44:12.277Z27/30 Write block 453 len 8192 data: 6 7
59052024-02-20T22:44:12.277Z28/30 Read block 268 len 8192
59062024-02-20T22:44:12.277Z29/30 Read block 404 len 20480
59072024-02-20T22:44:12.277Z30/30 Read block 332 len 8192
59082024-02-20T22:44:12.277Z----------------------------------------------------------------
59092024-02-20T22:44:12.277Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:29
59102024-02-20T22:44:12.277ZStates: Active Active Active
59112024-02-20T22:44:12.277ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59122024-02-20T22:44:12.277Z 2 Acked 1001 Write 4 Done Done Done false
59132024-02-20T22:44:12.277Z 3 Acked 1002 Read 10 Done Done Done false
59142024-02-20T22:44:12.277Z 4 Acked 1003 Write 6 Done Done Done false
59152024-02-20T22:44:12.277Z 5 Acked 1004 Read 4 Done Done Done false
59162024-02-20T22:44:12.277Z 6 Acked 1005 Read 3 Done Done Done false
59172024-02-20T22:44:12.277Z 7 Acked 1006 Write 9 Done Done Done false
59182024-02-20T22:44:12.277Z 8 Acked 1007 Read 4 Done Done Done false
59192024-02-20T22:44:12.277Z 9 Acked 1008 Read 9 Done Done Done false
59202024-02-20T22:44:12.277Z 10 Acked 1009 Write 4 Done Done Done false
59212024-02-20T22:44:12.277Z 11 Acked 1010 Read 6 Done Done Sent false
59222024-02-20T22:44:12.277Z 12 Acked 1011 Write 7 Done Done Sent false
59232024-02-20T22:44:12.277Z 13 Acked 1012 Write 10 Done Done Sent false
59242024-02-20T22:44:12.277Z 14 Acked 1013 Write 5 Done Done Sent false
59252024-02-20T22:44:12.278Z 15 Acked 1014 Read 8 Done Done Sent false
59262024-02-20T22:44:12.278Z 16 Acked 1015 Read 6 Done Done Sent false
59272024-02-20T22:44:12.278Z 17 Acked 1016 Read 1 Done Done Sent false
59282024-02-20T22:44:12.278Z 18 Acked 1017 Read 4 Done Done Sent false
59292024-02-20T22:44:12.278Z 19 Acked 1018 Write 7 Done Done Sent false
59302024-02-20T22:44:12.278Z 20 Acked 1019 Read 2 Done Done Sent false
59312024-02-20T22:44:12.278Z 21 Acked 1020 Write 4 Done Done Sent false
59322024-02-20T22:44:12.278Z 22 Acked 1021 Write 6 Done Done Sent false
59332024-02-20T22:44:12.278Z 23 Acked 1022 Read 5 Done Done Sent false
59342024-02-20T22:44:12.278Z 24 Acked 1023 Write 6 Done Done Sent false
59352024-02-20T22:44:12.278Z 25 Acked 1024 Read 3 Done Done Sent false
59362024-02-20T22:44:12.278Z 26 Acked 1025 Read 7 Done Done Sent false
59372024-02-20T22:44:12.278Z 27 Acked 1026 Write 2 Done Done Sent false
59382024-02-20T22:44:12.278Z 28 Acked 1027 Read 2 Done Done Sent false
59392024-02-20T22:44:12.278Z 29 Acked 1028 Read 5 Done Done Sent false
59402024-02-20T22:44:12.278Z 30 Acked 1029 Read 2 Done Sent Sent false
59412024-02-20T22:44:12.278Z STATES DS:0 DS:1 DS:2 TOTAL
59422024-02-20T22:44:12.278Z New 0 0 0 0
59432024-02-20T22:44:12.278Z Sent 0 1 20 21
59442024-02-20T22:44:12.278Z Done 29 28 9 66
59452024-02-20T22:44:12.278Z Skipped 0 0 0 0
59462024-02-20T22:44:12.278Z Error 0 0 0 0
59472024-02-20T22:44:12.278ZLast Flush: 1000 1000 1000
59482024-02-20T22:44:12.278ZDownstairs last five completed: 1000
59492024-02-20T22:44:12.278ZUpstairs last five completed: 30 29 28 27 26
59502024-02-20T22:44:12.278Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:11.802429238Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1351}
59512024-02-20T22:44:12.279ZWrote out file "/tmp/test_repair_verify.data"
59522024-02-20T22:44:12.279Z
59532024-02-20T22:44:12.283ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59542024-02-20T22:44:12.297ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59552024-02-20T22:44:12.298Z 0 000-019 52 52 52 98 98 98 F F F
59562024-02-20T22:44:12.298Z 1 020-039 52 52 52 98 98 98 T T T
59572024-02-20T22:44:12.298Z 2 040-059 52 52 52 98 98 98 T T F <---
59582024-02-20T22:44:12.298Z 3 060-079 50 50 50 94 94 94 F F F
59592024-02-20T22:44:12.298Z 4 080-099 37 37 37 66 66 66 F F F
59602024-02-20T22:44:12.298Z 5 100-119 52 52 52 98 98 98 F F F
59612024-02-20T22:44:12.298Z 6 120-139 52 52 52 98 98 98 T T F <---
59622024-02-20T22:44:12.298Z 7 140-159 51 51 51 97 97 97 F F F
59632024-02-20T22:44:12.298Z 8 160-179 47 47 47 88 88 88 T T F <---
59642024-02-20T22:44:12.303Z 9 180-199 48 48 48 89 89 89 T T F <---
59652024-02-20T22:44:12.304Z 10 200-219 41 41 41 76 76 76 T T F <---
59662024-02-20T22:44:12.304Z 11 220-239 53 53 53 99 99 99 F F F
59672024-02-20T22:44:12.304Z 12 240-259 47 47 47 88 88 88 T T F <---
59682024-02-20T22:44:12.304Z 13 260-279 50 50 50 96 96 96 F F F
59692024-02-20T22:44:12.304Z 14 280-299 50 50 50 96 96 96 F F F
59702024-02-20T22:44:12.304Z 15 300-319 47 47 47 88 88 88 T T T
59712024-02-20T22:44:12.304Z 16 320-339 45 45 45 85 85 85 T T T
59722024-02-20T22:44:12.304Z 17 340-359 51 51 51 97 97 97 F F F
59732024-02-20T22:44:12.304Z 18 360-379 51 51 51 97 97 97 F F F
59742024-02-20T22:44:12.304Z 19 380-399 48 48 48 90 90 90 F F F
59752024-02-20T22:44:12.304Z 20 400-419 51 51 51 97 97 97 F F F
59762024-02-20T22:44:12.304Z 21 420-439 52 52 52 98 98 98 F F F
59772024-02-20T22:44:12.304Z 22 440-459 53 53 53 99 99 99 T T T
59782024-02-20T22:44:12.305Z 23 460-479 53 53 53 99 99 99 T T F <---
59792024-02-20T22:44:12.305Z 24 480-499 53 53 53 99 99 99 F F F
59802024-02-20T22:44:12.305Z 25 500-519 52 52 52 98 98 98 F F F
59812024-02-20T22:44:12.305Z 26 520-539 53 53 53 99 99 99 F F F
59822024-02-20T22:44:12.305Z 27 540-559 52 52 52 98 98 98 F F F
59832024-02-20T22:44:12.305Z 28 560-579 53 53 53 99 99 99 F F F
59842024-02-20T22:44:12.305Z 29 580-599 52 52 52 98 98 98 T T T
59852024-02-20T22:44:12.305ZMax gen: 53, Max flush: 99
59862024-02-20T22:44:12.305ZError: Difference in extent metadata found!
59872024-02-20T22:44:12.305ZA Difference in extent metadata is expected here
59882024-02-20T22:44:12.305ZOn loop 27
59892024-02-20T22:44:12.305Z
59902024-02-20T22:44:12.305ZVerifying data now
59912024-02-20T22:44:17.927ZNo change after verify
59922024-02-20T22:44:17.931ZLoop: 27 Downstairs dump after verify (and repair):
59932024-02-20T22:44:17.942ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59942024-02-20T22:44:17.942Z 0 000-019 52 52 52 98 98 98 F F F
59952024-02-20T22:44:17.955Z 1 020-039 54 54 54 100 100 100 F F F
59962024-02-20T22:44:17.956Z 2 040-059 54 54 54 100 100 100 F F F
59972024-02-20T22:44:17.956Z 3 060-079 50 50 50 94 94 94 F F F
59982024-02-20T22:44:17.956Z 4 080-099 37 37 37 66 66 66 F F F
59992024-02-20T22:44:17.956Z 5 100-119 52 52 52 98 98 98 F F F
60002024-02-20T22:44:17.956Z 6 120-139 54 54 54 100 100 100 F F F
60012024-02-20T22:44:17.956Z 7 140-159 51 51 51 97 97 97 F F F
60022024-02-20T22:44:17.956Z 8 160-179 54 54 54 100 100 100 F F F
60032024-02-20T22:44:17.956Z 9 180-199 54 54 54 100 100 100 F F F
60042024-02-20T22:44:17.956Z 10 200-219 54 54 54 100 100 100 F F F
60052024-02-20T22:44:17.956Z 11 220-239 53 53 53 99 99 99 F F F
60062024-02-20T22:44:17.956Z 12 240-259 54 54 54 100 100 100 F F F
60072024-02-20T22:44:17.956Z 13 260-279 50 50 50 96 96 96 F F F
60082024-02-20T22:44:17.956Z 14 280-299 50 50 50 96 96 96 F F F
60092024-02-20T22:44:17.957Z 15 300-319 54 54 54 100 100 100 F F F
60102024-02-20T22:44:17.957Z 16 320-339 54 54 54 100 100 100 F F F
60112024-02-20T22:44:17.957Z 17 340-359 51 51 51 97 97 97 F F F
60122024-02-20T22:44:17.957Z 18 360-379 51 51 51 97 97 97 F F F
60132024-02-20T22:44:17.957Z 19 380-399 48 48 48 90 90 90 F F F
60142024-02-20T22:44:17.957Z 20 400-419 51 51 51 97 97 97 F F F
60152024-02-20T22:44:17.957Z 21 420-439 52 52 52 98 98 98 F F F
60162024-02-20T22:44:17.957Z 22 440-459 54 54 54 100 100 100 F F F
60172024-02-20T22:44:17.958Z 23 460-479 54 54 54 100 100 100 F F F
60182024-02-20T22:44:17.958Z 24 480-499 53 53 53 99 99 99 F F F
60192024-02-20T22:44:17.958Z 25 500-519 52 52 52 98 98 98 F F F
60202024-02-20T22:44:17.958Z 26 520-539 53 53 53 99 99 99 F F F
60212024-02-20T22:44:17.958Z 27 540-559 52 52 52 98 98 98 F F F
60222024-02-20T22:44:17.958Z 28 560-579 53 53 53 99 99 99 F F F
60232024-02-20T22:44:17.958Z 29 580-599 54 54 54 100 100 100 F F F
60242024-02-20T22:44:17.958ZMax gen: 54, Max flush: 100
60252024-02-20T22:44:17.958Z
60262024-02-20T22:44:17.958ZBegin loop 28
60272024-02-20T22:44:17.958ZDownstairs to restart: 0
60282024-02-20T22:44:18.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.4936423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359}
60292024-02-20T22:44:18.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.493962768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359}
60302024-02-20T22:44:18.807Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494012246Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359}
60312024-02-20T22:44:18.807ZCrucible runtime is spawned
60322024-02-20T22:44:18.807Z{"msg":"Crucible 04aaa4d7-f869-4e5d-8af7-dff4aededde7 has session id: 7fa5bd3a-97d4-46c5-b271-aab147c50abe","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494043275Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60332024-02-20T22:44:18.807Z{"msg":"Upstairs opts: Upstairs UUID: 04aaa4d7-f869-4e5d-8af7-dff4aededde7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494074144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60342024-02-20T22:44:18.807Z{"msg":"Crucible stats registered with UUID: 04aaa4d7-f869-4e5d-8af7-dff4aededde7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494096163Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60352024-02-20T22:44:18.807Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494131162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359}
60362024-02-20T22:44:18.807Z{"msg":"04aaa4d7-f869-4e5d-8af7-dff4aededde7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494291066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60372024-02-20T22:44:18.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494309376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60382024-02-20T22:44:18.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494324455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60392024-02-20T22:44:18.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494352784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60402024-02-20T22:44:18.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494367234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60412024-02-20T22:44:18.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494396603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60422024-02-20T22:44:18.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494431961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60432024-02-20T22:44:18.807Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.49445943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60442024-02-20T22:44:18.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494498629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60452024-02-20T22:44:18.807Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494523468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60462024-02-20T22:44:18.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494723791Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60472024-02-20T22:44:18.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494783009Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60482024-02-20T22:44:18.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.494813668Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60492024-02-20T22:44:18.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.495383417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60502024-02-20T22:44:18.807Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.495423016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60512024-02-20T22:44:18.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.495439555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60522024-02-20T22:44:18.807Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.495453165Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60532024-02-20T22:44:18.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.495532752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60542024-02-20T22:44:18.808Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.49558528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60552024-02-20T22:44:18.808Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.49614764Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60562024-02-20T22:44:18.808Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496183849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60572024-02-20T22:44:18.808Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496201128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60582024-02-20T22:44:18.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496636862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60592024-02-20T22:44:18.808Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496679461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60602024-02-20T22:44:18.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.49670309Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60612024-02-20T22:44:18.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496803236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60622024-02-20T22:44:18.808Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496830786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60632024-02-20T22:44:18.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496845765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60642024-02-20T22:44:18.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496858825Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60652024-02-20T22:44:18.808Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496879764Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60662024-02-20T22:44:18.808Z{"msg":"[0]R flush_numbers[0..12]: [98, 100, 100, 94, 66, 98, 100, 97, 100, 100, 100, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496894243Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60672024-02-20T22:44:18.808Z{"msg":"[0]R generation[0..12]: [52, 54, 54, 50, 37, 52, 54, 51, 54, 54, 54, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496908443Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60682024-02-20T22:44:18.808Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496930312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60692024-02-20T22:44:18.808Z{"msg":"[1]R flush_numbers[0..12]: [98, 100, 100, 94, 66, 98, 100, 97, 100, 100, 100, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496945082Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60702024-02-20T22:44:18.808Z{"msg":"[1]R generation[0..12]: [52, 54, 54, 50, 37, 52, 54, 51, 54, 54, 54, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496958741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60712024-02-20T22:44:18.808Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.496972681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60722024-02-20T22:44:18.808Z{"msg":"[2]R flush_numbers[0..12]: [98, 100, 100, 94, 66, 98, 100, 97, 100, 100, 100, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.49698817Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60732024-02-20T22:44:18.808Z{"msg":"[2]R generation[0..12]: [52, 54, 54, 50, 37, 52, 54, 51, 54, 54, 54, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497009709Z","hostname":"Wait for a query_work_queue command to finish before sending IO
60742024-02-20T22:44:18.808Zip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60752024-02-20T22:44:18.808Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497028479Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60762024-02-20T22:44:18.808Z{"msg":"Max found gen is 55","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497048418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60772024-02-20T22:44:18.808Z{"msg":"Generation requested: 56 >= found:55","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497063067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60782024-02-20T22:44:18.809Z{"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497088046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60792024-02-20T22:44:18.809Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497101646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60802024-02-20T22:44:18.809Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497126665Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60812024-02-20T22:44:18.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497139004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60822024-02-20T22:44:18.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497152574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60832024-02-20T22:44:18.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497165614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60842024-02-20T22:44:18.809Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497199312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60852024-02-20T22:44:18.809Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497213442Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60862024-02-20T22:44:18.809Z{"msg":"04aaa4d7-f869-4e5d-8af7-dff4aededde7 is now active with session: 7fa5bd3a-97d4-46c5-b271-aab147c50abe","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497234021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60872024-02-20T22:44:18.809Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.497246331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359,"session_id":"7fa5bd3a-97d4-46c5-b271-aab147c50abe"}
60882024-02-20T22:44:18.809Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:17.49726527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359}
60892024-02-20T22:44:18.809ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60902024-02-20T22:44:18.809ZLoading write count information from file "/tmp/test_repair_verify.data"
60912024-02-20T22:44:18.809ZRun Repair workload
60922024-02-20T22:44:18.809Z01/30 Write block 591 len 32768 data: 8 7 7 8 8 7 7 5
60932024-02-20T22:44:18.809Z02/30 Read block 300 len 4096
60942024-02-20T22:44:18.809Z03/30 Write block 148 len 20480 data: 3 5 7 7 6
60952024-02-20T22:44:18.809Z04/30 Read block 88 len 8192
60962024-02-20T22:44:18.809Z05/30 Write block 389 len 32768 data: 6 6 5 7 7 7 7 8
60972024-02-20T22:44:18.809Z06/30 Read block 556 len 20480
60982024-02-20T22:44:18.809Z07/30 Read block 476 len 20480
60992024-02-20T22:44:18.809Z08/30 Write block 471 len 28672 data: 5 5 5 6 6 6 8
61002024-02-20T22:44:18.809Z09/30 Read block 478 len 4096
61012024-02-20T22:44:18.809Z10/30 Write block 60 len 4096 data: 3
61022024-02-20T22:44:18.809Z11/30 Read block 279 len 8192
61032024-02-20T22:44:18.810Z12/30 Write block 590 len 24576 data: 9 9 8 8 9 9
61042024-02-20T22:44:18.810Z13/30 Read block 159 len 24576
61052024-02-20T22:44:18.810Z14/30 Write block 11 len 12288 data: 4 2 2
61062024-02-20T22:44:18.810Z15/30 Write block 221 len 28672 data: 3 4 4 3 3 3 4
61072024-02-20T22:44:18.810Z16/30 Write block 455 len 36864 data: 8 9 10 10 8 7 7 9 9
61082024-02-20T22:44:18.810Z17/30 Write block 1 len 40960 data: 2 4 4 5 6 5 4 4 4 5
61092024-02-20T22:44:18.810Z18/30 Write block 419 len 20480 data: 6 6 5 4 4
61102024-02-20T22:44:18.810Z19/30 Read block 567 len 40960
61112024-02-20T22:44:18.810Z20/30 Read block 243 len 24576
61122024-02-20T22:44:18.810Z21/30 Read block 233 len 32768
61132024-02-20T22:44:18.810Z22/30 Write block 13 len 4096 data: 3
61142024-02-20T22:44:18.810Z23/30 Read block 401 len 32768
61152024-02-20T22:44:18.810Z24/30 Read block 341 len 24576
61162024-02-20T22:44:18.810Z25/30 Write block 81 len 36864 data: 4 3 3 4 3 3 3 3 3
61172024-02-20T22:44:18.810Z26/30 Write block 362 len 40960 data: 9 12 10 13 11 7 6 5 7 8
61182024-02-20T22:44:18.810Z27/30 Write block 120 len 24576 data: 6 5 5 4 3 3
61192024-02-20T22:44:18.810Z28/30 Read block 93 len 24576
61202024-02-20T22:44:18.810Z29/30 Read block 468 len 4096
61212024-02-20T22:44:18.810Z30/30 Read block 41 len 24576
61222024-02-20T22:44:18.810Z----------------------------------------------------------------
61232024-02-20T22:44:18.810Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61242024-02-20T22:44:18.810ZStates: Active Active Active
61252024-02-20T22:44:18.810ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61262024-02-20T22:44:18.810Z 1 Acked 1000 Write 8 Done Done Done false
61272024-02-20T22:44:18.810Z 2 Acked 1001 Read 1 Done Done Done false
61282024-02-20T22:44:18.818Z 3 Acked 1002 Write 5 Sent Done Done false
61292024-02-20T22:44:18.818Z 4 Acked 1003 Read 2 Sent Done Done false
61302024-02-20T22:44:18.818Z 5 Acked 1004 Write 8 Sent Done Done false
61312024-02-20T22:44:18.818Z 6 Acked 1005 Read 5 Sent Done Done false
61322024-02-20T22:44:18.818Z 7 Acked 1006 Read 5 Sent Done Done false
61332024-02-20T22:44:18.818Z 8 Acked 1007 Write 7 Sent Done Done false
61342024-02-20T22:44:18.818Z 9 Acked 1008 Read 1 Sent Done Done false
61352024-02-20T22:44:18.818Z 10 Acked 1009 Write 1 Sent Done Done false
61362024-02-20T22:44:18.818Z 11 Acked 1010 Read 2 Sent Done Done false
61372024-02-20T22:44:18.818Z 12 Acked 1011 Write 6 Sent Done Done false
61382024-02-20T22:44:18.818Z 13 Acked 1012 Read 6 Sent Done Done false
61392024-02-20T22:44:18.818Z 14 Acked 1013 Write 3 Sent Done Done false
61402024-02-20T22:44:18.818Z 15 Acked 1014 Write 7 Sent Done Done false
61412024-02-20T22:44:18.818Z 16 Acked 1015 Write 9 Sent Done Done false
61422024-02-20T22:44:18.819Z 17 Acked 1016 Write 10 Sent Done Done false
61432024-02-20T22:44:18.819Z 18 Acked 1017 Write 5 Sent Done Done false
61442024-02-20T22:44:18.819Z 19 Acked 1018 Read 10 Sent Done Done false
61452024-02-20T22:44:18.819Z 20 Acked 1019 Read 6 Sent Done Done false
61462024-02-20T22:44:18.819Z 21 Acked 1020 Read 8 Sent Done Done false
61472024-02-20T22:44:18.819Z 22 Acked 1021 Write 1 Sent Done Done false
61482024-02-20T22:44:18.819Z 23 Acked 1022 Read 8 Sent Done Done false
61492024-02-20T22:44:18.819Z 24 Acked 1023 Read 6 Sent Done Done false
61502024-02-20T22:44:18.819Z 25 Acked 1024 Write 9 Sent Done Done false
61512024-02-20T22:44:18.819Z 26 Acked 1025 Write 10 Sent Done Done false
61522024-02-20T22:44:18.819Z 27 Acked 1026 Write 6 Sent Done Done false
61532024-02-20T22:44:18.819Z 28 Acked 1027 Read 6 Sent Done Done false
61542024-02-20T22:44:18.819Z 29 Acked 1028 Read 1 Sent Done Done false
61552024-02-20T22:44:18.819Z 30 Acked 1029 Read 6 Sent Sent Done false
61562024-02-20T22:44:18.819Z STATES DS:0 DS:1 DS:2 TOTAL
61572024-02-20T22:44:18.819Z New 0 0 0 0
61582024-02-20T22:44:18.819Z Sent 28 1 0 29
61592024-02-20T22:44:18.819Z Done 2 29 30 61
61602024-02-20T22:44:18.819Z Skipped 0 0 0 0
61612024-02-20T22:44:18.819Z Error 0 0 0 0
61622024-02-20T22:44:18.819ZLast Flush: 0 0 0
61632024-02-20T22:44:18.820ZDownstairs last five completed:
61642024-02-20T22:44:18.820ZUpstairs last five completed: 30 29 28 27 26
61652024-02-20T22:44:18.820Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:17.525162202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1359}
61662024-02-20T22:44:18.820ZWrote out file "/tmp/test_repair_verify.data"
61672024-02-20T22:44:18.820Z
61682024-02-20T22:44:18.820ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61692024-02-20T22:44:18.820ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61702024-02-20T22:44:18.820Z 0 000-019 52 52 52 98 98 98 F T T <---
61712024-02-20T22:44:18.820Z 1 020-039 54 54 54 100 100 100 F F F
61722024-02-20T22:44:18.820Z 2 040-059 54 54 54 100 100 100 F F F
61732024-02-20T22:44:18.820Z 3 060-079 50 50 50 94 94 94 F T T <---
61742024-02-20T22:44:18.820Z 4 080-099 37 37 37 66 66 66 F T T <---
61752024-02-20T22:44:18.820Z 5 100-119 52 52 52 98 98 98 F F F
61762024-02-20T22:44:18.820Z 6 120-139 54 54 54 100 100 100 F T T <---
61772024-02-20T22:44:18.820Z 7 140-159 51 51 51 97 97 97 F T T <---
61782024-02-20T22:44:18.820Z 8 160-179 54 54 54 100 100 100 F F F
61792024-02-20T22:44:18.820Z 9 180-199 54 54 54 100 100 100 F F F
61802024-02-20T22:44:18.820Z 10 200-219 54 54 54 100 100 100 F F F
61812024-02-20T22:44:18.820Z 11 220-239 53 53 53 99 99 99 F T T <---
61822024-02-20T22:44:18.820Z 12 240-259 54 54 54 100 100 100 F F F
61832024-02-20T22:44:18.820Z 13 260-279 50 50 50 96 96 96 F F F
61842024-02-20T22:44:18.820Z 14 280-299 50 50 50 96 96 96 F F F
61852024-02-20T22:44:18.820Z 15 300-319 54 54 54 100 100 100 F F F
61862024-02-20T22:44:18.821Z 16 320-339 54 54 54 100 100 100 F F F
61872024-02-20T22:44:18.821Z 17 340-359 51 51 51 97 97 97 F F F
61882024-02-20T22:44:18.821Z 18 360-379 51 51 51 97 97 97 F T T <---
61892024-02-20T22:44:18.821Z 19 380-399 48 48 48 90 90 90 F T T <---
61902024-02-20T22:44:18.821Z 20 400-419 51 51 51 97 97 97 F T T <---
61912024-02-20T22:44:18.821Z 21 420-439 52 52 52 98 98 98 F T T <---
61922024-02-20T22:44:18.821Z 22 440-459 54 54 54 100 100 100 F T T <---
61932024-02-20T22:44:18.821Z 23 460-479 54 54 54 100 100 100 F T T <---
61942024-02-20T22:44:18.821Z 24 480-499 53 53 53 99 99 99 F F F
61952024-02-20T22:44:18.821Z 25 500-519 52 52 52 98 98 98 F F F
61962024-02-20T22:44:18.821Z 26 520-539 53 53 53 99 99 99 F F F
61972024-02-20T22:44:18.821Z 27 540-559 52 52 52 98 98 98 F F F
61982024-02-20T22:44:18.821Z 28 560-579 53 53 53 99 99 99 F F F
61992024-02-20T22:44:18.821Z 29 580-599 54 54 54 100 100 100 T T T
62002024-02-20T22:44:18.821ZMax gen: 54, Max flush: 100
62012024-02-20T22:44:18.821ZError: Difference in extent metadata found!
62022024-02-20T22:44:18.821ZA Difference in extent metadata is expected here
62032024-02-20T22:44:18.821ZOn loop 28
62042024-02-20T22:44:18.821Z
62052024-02-20T22:44:18.821ZVerifying data now
62062024-02-20T22:44:22.699ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
62072024-02-20T22:44:22.704Z592a593,594
62082024-02-20T22:44:22.704Z> 9,
62092024-02-20T22:44:22.704Z> 9,
62102024-02-20T22:44:22.704Z594a597,598
62112024-02-20T22:44:22.704Z> 9,
62122024-02-20T22:44:22.704Z> 9,
62132024-02-20T22:44:22.704Z597,600d600
62142024-02-20T22:44:22.704Z< 8,
62152024-02-20T22:44:22.704Z< 8,
62162024-02-20T22:44:22.705Z< 7,
62172024-02-20T22:44:22.705Z< 7,
62182024-02-20T22:44:22.705Zdiff found after verify
62192024-02-20T22:44:22.709ZLoop: 28 Downstairs dump after verify (and repair):
62202024-02-20T22:44:22.720ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62212024-02-20T22:44:22.720Z 0 000-019 55 55 55 101 101 101 F F F
62222024-02-20T22:44:22.720Z 1 020-039 54 54 54 100 100 100 F F F
62232024-02-20T22:44:22.720Z 2 040-059 54 54 54 100 100 100 F F F
62242024-02-20T22:44:22.720Z 3 060-079 55 55 55 101 101 101 F F F
62252024-02-20T22:44:22.720Z 4 080-099 55 55 55 101 101 101 F F F
62262024-02-20T22:44:22.720Z 5 100-119 52 52 52 98 98 98 F F F
62272024-02-20T22:44:22.725Z 6 120-139 55 55 55 101 101 101 F F F
62282024-02-20T22:44:22.725Z 7 140-159 55 55 55 101 101 101 F F F
62292024-02-20T22:44:22.725Z 8 160-179 54 54 54 100 100 100 F F F
62302024-02-20T22:44:22.725Z 9 180-199 54 54 54 100 100 100 F F F
62312024-02-20T22:44:22.725Z 10 200-219 54 54 54 100 100 100 F F F
62322024-02-20T22:44:22.725Z 11 220-239 55 55 55 101 101 101 F F F
62332024-02-20T22:44:22.725Z 12 240-259 54 54 54 100 100 100 F F F
62342024-02-20T22:44:22.725Z 13 260-279 50 50 50 96 96 96 F F F
62352024-02-20T22:44:22.725Z 14 280-299 50 50 50 96 96 96 F F F
62362024-02-20T22:44:22.725Z 15 300-319 54 54 54 100 100 100 F F F
62372024-02-20T22:44:22.725Z 16 320-339 54 54 54 100 100 100 F F F
62382024-02-20T22:44:22.726Z 17 340-359 51 51 51 97 97 97 F F F
62392024-02-20T22:44:22.726Z 18 360-379 55 55 55 101 101 101 F F F
62402024-02-20T22:44:22.726Z 19 380-399 55 55 55 101 101 101 F F F
62412024-02-20T22:44:22.726Z 20 400-419 55 55 55 101 101 101 F F F
62422024-02-20T22:44:22.726Z 21 420-439 55 55 55 101 101 101 F F F
62432024-02-20T22:44:22.726Z 22 440-459 55 55 55 101 101 101 F F F
62442024-02-20T22:44:22.726Z 23 460-479 55 55 55 101 101 101 F F F
62452024-02-20T22:44:22.726Z 24 480-499 53 53 53 99 99 99 F F F
62462024-02-20T22:44:22.726Z 25 500-519 52 52 52 98 98 98 F F F
62472024-02-20T22:44:22.726Z 26 520-539 53 53 53 99 99 99 F F F
62482024-02-20T22:44:22.726Z 27 540-559 52 52 52 98 98 98 F F F
62492024-02-20T22:44:22.726Z 28 560-579 53 53 53 99 99 99 F F F
62502024-02-20T22:44:22.726Z 29 580-599 55 55 55 101 101 101 F F F
62512024-02-20T22:44:22.726ZMax gen: 55, Max flush: 101
62522024-02-20T22:44:22.726Z
62532024-02-20T22:44:22.726ZBegin loop 29
62542024-02-20T22:44:22.726ZDownstairs to restart: 1
62552024-02-20T22:44:22.742Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.27191334Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370}
62562024-02-20T22:44:22.742Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.27219741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370}
62572024-02-20T22:44:22.742Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272227718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370}
62582024-02-20T22:44:22.752Z{"msg":"Crucible 95d91a2a-6c7d-4d79-9d1c-74084d282f52 has session id: 8b92243a-c46f-4af9-bd5e-597b3f57ae30","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
62592024-02-20T22:44:22.752Ztime":"2024-02-20T22:44:22.272258607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62602024-02-20T22:44:22.752Z{"msg":"Upstairs opts: Upstairs UUID: 95d91a2a-6c7d-4d79-9d1c-74084d282f52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272321765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62612024-02-20T22:44:22.752Z{"msg":"Crucible stats registered with UUID: 95d91a2a-6c7d-4d79-9d1c-74084d282f52","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272380473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62622024-02-20T22:44:22.752Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272411752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370}
62632024-02-20T22:44:22.753Z{"msg":"95d91a2a-6c7d-4d79-9d1c-74084d282f52 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272492639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62642024-02-20T22:44:22.753Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272529367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62652024-02-20T22:44:22.753Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272585416Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62662024-02-20T22:44:22.753Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272632414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62672024-02-20T22:44:22.753Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272664573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62682024-02-20T22:44:22.753Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272696861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62692024-02-20T22:44:22.753Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272891865Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62702024-02-20T22:44:22.753Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272940993Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62712024-02-20T22:44:22.753Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272971822Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62722024-02-20T22:44:22.753Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.272997641Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62732024-02-20T22:44:22.753Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.27302367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62742024-02-20T22:44:22.753Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.273072878Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62752024-02-20T22:44:22.754Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.273099717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62762024-02-20T22:44:22.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.273815532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62772024-02-20T22:44:22.754Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.273884439Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62782024-02-20T22:44:22.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.273945037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62792024-02-20T22:44:22.754Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.273979736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62802024-02-20T22:44:22.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274004015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62812024-02-20T22:44:22.754Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274026154Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62822024-02-20T22:44:22.754Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274435069Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62832024-02-20T22:44:22.754Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274476418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62842024-02-20T22:44:22.754Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274598124Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62852024-02-20T22:44:22.754Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274861474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62862024-02-20T22:44:22.754Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274899493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62872024-02-20T22:44:22.754Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.274926562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62882024-02-20T22:44:22.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.27496216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62892024-02-20T22:44:22.755Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275014169Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62902024-02-20T22:44:22.755Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275031068Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62912024-02-20T22:44:22.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275052967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62922024-02-20T22:44:22.755Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275067147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62932024-02-20T22:44:22.755Z{"msg":"[0]R flush_numbers[0..12]: [101, 100, 100, 101, 101, 98, 101, 101, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275089136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62942024-02-20T22:44:22.755Z{"msg":"[0]R generation[0..12]: [55, 54, 54, 55, 55, 52, 55, 55, 54, 54, 54, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275103775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62952024-02-20T22:44:22.755Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275124925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62962024-02-20T22:44:22.755Z{"msg":"[1]R flush_numbers[0..12]: [101, 100, 100, 101, 101, 98, 101, 101, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275140324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62972024-02-20T22:44:22.755Z{"msg":"[1]R generation[0..12]: [55, 54, 54, 55, 55, 52, 55, 55, 54, 54, 54, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275161063Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62982024-02-20T22:44:22.755Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275175553Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
62992024-02-20T22:44:22.755Z{"msg":"[2]R flush_numbers[0..12]: [101, 100, 100, 101, 101, 98, 101, 101, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275202292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63002024-02-20T22:44:22.755Z{"msg":"[2]R generation[0..12]: [55, 54, 54, 55, 55, 52, 55, 55, 54, 54, 54, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275216391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63012024-02-20T22:44:22.756Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275230511Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63022024-02-20T22:44:22.756Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.27524489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63032024-02-20T22:44:22.756Z{Wait for a query_work_queue command to finish before sending IO
63042024-02-20T22:44:22.756Z"msg":"Generation requested: 58 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.27526507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63052024-02-20T22:44:22.756Z{"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275285429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63062024-02-20T22:44:22.756Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275299148Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63072024-02-20T22:44:22.756Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275314818Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63082024-02-20T22:44:22.756Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275327327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63092024-02-20T22:44:22.756Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275348837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63102024-02-20T22:44:22.756Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275362646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63112024-02-20T22:44:22.756Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275382585Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63122024-02-20T22:44:22.756Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275395445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63132024-02-20T22:44:22.757Z{"msg":"95d91a2a-6c7d-4d79-9d1c-74084d282f52 is now active with session: 8b92243a-c46f-4af9-bd5e-597b3f57ae30","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275416314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63142024-02-20T22:44:22.757Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275428814Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370,"session_id":"8b92243a-c46f-4af9-bd5e-597b3f57ae30"}
63152024-02-20T22:44:22.757Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:22.275441773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370}
63162024-02-20T22:44:22.757ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63172024-02-20T22:44:22.757ZLoading write count information from file "/tmp/test_repair_verify.data"
63182024-02-20T22:44:22.757ZRun Repair workload
63192024-02-20T22:44:22.757Z01/30 Write block 456 len 36864 data: 10 11 11 9 8 8 10 10 8
63202024-02-20T22:44:22.757Z02/30 Flush
63212024-02-20T22:44:22.764Z03/30 Flush
63222024-02-20T22:44:22.764Z04/30 Write block 285 len 28672 data: 7 8 8 7 6 4 4
63232024-02-20T22:44:22.764Z05/30 Read block 480 len 4096
63242024-02-20T22:44:22.764Z06/30 Flush
63252024-02-20T22:44:22.764Z07/30 Write block 119 len 16384 data: 5 7 6 6
63262024-02-20T22:44:22.764Z08/30 Read block 278 len 24576
63272024-02-20T22:44:22.764Z09/30 Write block 473 len 12288 data: 6 7 7
63282024-02-20T22:44:22.764Z10/30 Write block 48 len 20480 data: 4 4 3 4 6
63292024-02-20T22:44:22.764Z11/30 Read block 70 len 12288
63302024-02-20T22:44:22.764Z12/30 Write block 197 len 40960 data: 6 5 6 5 5 5 4 4 5 5
63312024-02-20T22:44:22.765Z13/30 Read block 64 len 4096
63322024-02-20T22:44:22.765Z14/30 Read block 525 len 20480
63332024-02-20T22:44:22.765Z15/30 Read block 159 len 16384
63342024-02-20T22:44:22.765Z16/30 Write block 23 len 12288 data: 4 3 7
63352024-02-20T22:44:22.774Z17/30 Write block 520 len 8192 data: 2 3
63362024-02-20T22:44:22.774Z18/30 Read block 221 len 24576
63372024-02-20T22:44:22.774Z19/30 Read block 448 len 40960
63382024-02-20T22:44:22.774Z20/30 Read block 325 len 40960
63392024-02-20T22:44:22.774Z21/30 Write block 25 len 20480 data: 8 7 11 12 15
63402024-02-20T22:44:22.774Z22/30 Read block 295 len 40960
63412024-02-20T22:44:22.774Z23/30 Read block 92 len 32768
63422024-02-20T22:44:22.774Z24/30 Write block 546 len 12288 data: 4 6 6
63432024-02-20T22:44:22.780Z25/30 Write block 541 len 20480 data: 5 5 5 4 5
63442024-02-20T22:44:22.780Z26/30 Flush
63452024-02-20T22:44:22.780Z27/30 Write block 533 len 36864 data: 5 8 8 8 6 7 5 5 6
63462024-02-20T22:44:22.780Z28/30 Read block 171 len 12288
63472024-02-20T22:44:22.780Z29/30 Write block 491 len 20480 data: 2 4 6 7 7
63482024-02-20T22:44:22.780Z30/30 Write block 71 len 16384 data: 6 6 4 5
63492024-02-20T22:44:22.780Z----------------------------------------------------------------
63502024-02-20T22:44:22.780Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63512024-02-20T22:44:22.780ZStates: Active Active Active
63522024-02-20T22:44:22.780ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63532024-02-20T22:44:22.780Z 1 Acked 1000 Write 9 Done Sent Done false
63542024-02-20T22:44:22.780Z 2 Acked 1001 Flush 0 Done Sent Done false
63552024-02-20T22:44:22.780Z 3 Acked 1002 Flush 0 Done Sent Done false
63562024-02-20T22:44:22.780Z 4 Acked 1003 Write 7 Done Sent Done false
63572024-02-20T22:44:22.780Z 5 Acked 1004 Read 1 Done Sent Done false
63582024-02-20T22:44:22.780Z 6 Acked 1005 Flush 0 Done Sent Done false
63592024-02-20T22:44:22.780Z 7 Acked 1006 Write 4 Done Sent Done false
63602024-02-20T22:44:22.781Z 8 Acked 1007 Read 6 Done Sent Done false
63612024-02-20T22:44:22.781Z 9 Acked 1008 Write 3 Done Sent Done false
63622024-02-20T22:44:22.781Z 10 Acked 1009 Write 5 Done Sent Done false
63632024-02-20T22:44:22.781Z 11 Acked 1010 Read 3 Done Sent Done false
63642024-02-20T22:44:22.781Z 12 Acked 1011 Write 10 Done Sent Done false
63652024-02-20T22:44:22.781Z 13 Acked 1012 Read 1 Done Sent Done false
63662024-02-20T22:44:22.781Z 14 Acked 1013 Read 5 Done Sent Done false
63672024-02-20T22:44:22.781Z 15 Acked 1014 Read 4 Done Sent Done false
63682024-02-20T22:44:22.781Z 16 Acked 1015 Write 3 Done Sent Done false
63692024-02-20T22:44:22.781Z 17 Acked 1016 Write 2 Done Sent Done false
63702024-02-20T22:44:22.781Z 18 Acked 1017 Read 6 Done Sent Done false
63712024-02-20T22:44:22.781Z 19 Acked 1018 Read 10 Done Sent Done false
63722024-02-20T22:44:22.781Z 20 Acked 1019 Read 10 Done Sent Done false
63732024-02-20T22:44:22.781Z 21 Acked 1020 Write 5 Done Sent Done false
63742024-02-20T22:44:22.781Z 22 Acked 1021 Read 10 Done Sent Done false
63752024-02-20T22:44:22.781Z 23 Acked 1022 Read 8 Done Sent Done false
63762024-02-20T22:44:22.781Z 24 Acked 1023 Write 3 Done Sent Done false
63772024-02-20T22:44:22.781Z 25 Acked 1024 Write 5 Done Sent Done false
63782024-02-20T22:44:22.781Z 26 Acked 1025 Flush 0 Done Sent Done false
63792024-02-20T22:44:22.781Z 27 Acked 1026 Write 9 Done Sent Done false
63802024-02-20T22:44:22.781Z 28 Acked 1027 Read 3 Done Sent Done false
63812024-02-20T22:44:22.781Z 29 Acked 1028 Write 5 Sent Sent Sent false
63822024-02-20T22:44:22.781Z 30 Acked 1029 Write 4 Sent Sent Sent false
63832024-02-20T22:44:22.782Z STATES DS:0 DS:1 DS:2 TOTAL
63842024-02-20T22:44:22.782Z New 0 0 0 0
63852024-02-20T22:44:22.782Z Sent 2 30 2 34
63862024-02-20T22:44:22.782Z Done 28 0 28 56
63872024-02-20T22:44:22.782Z Skipped 0 0 0 0
63882024-02-20T22:44:22.782Z Error 0 0 0 0
63892024-02-20T22:44:22.782ZLast Flush: 1025 0 1025
63902024-02-20T22:44:22.782ZDownstairs last five completed:
63912024-02-20T22:44:22.782ZUpstairs last five completed: 30 29 28 27 26
63922024-02-20T22:44:22.782Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:22.309527484Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1370}
63932024-02-20T22:44:22.790ZWrote out file "/tmp/test_repair_verify.data"
63942024-02-20T22:44:22.790Z
63952024-02-20T22:44:22.790ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63962024-02-20T22:44:22.801ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63972024-02-20T22:44:22.801Z 0 000-019 55 55 55 101 101 101 F F F
63982024-02-20T22:44:22.801Z 1 020-039 58 54 58 105 100 105 F F F <---
63992024-02-20T22:44:22.801Z 2 040-059 58 54 58 105 100 105 F F F <---
64002024-02-20T22:44:22.801Z 3 060-079 55 55 55 101 101 101 T F T <---
64012024-02-20T22:44:22.801Z 4 080-099 55 55 55 101 101 101 F F F
64022024-02-20T22:44:22.801Z 5 100-119 58 52 58 105 98 105 F F F <---
64032024-02-20T22:44:22.801Z 6 120-139 58 55 58 105 101 105 F F F <---
64042024-02-20T22:44:22.802Z 7 140-159 55 55 55 101 101 101 F F F
64052024-02-20T22:44:22.802Z 8 160-179 54 54 54 100 100 100 F F F
64062024-02-20T22:44:22.802Z 9 180-199 58 54 58 105 100 105 F F F <---
64072024-02-20T22:44:22.802Z 10 200-219 58 54 58 105 100 105 F F F <---
64082024-02-20T22:44:22.807Z 11 220-239 55 55 55 101 101 101 F F F
64092024-02-20T22:44:22.807Z 12 240-259 54 54 54 100 100 100 F F F
64102024-02-20T22:44:22.807Z 13 260-279 50 50 50 96 96 96 F F F
64112024-02-20T22:44:22.807Z 14 280-299 58 50 58 104 96 104 F F F <---
64122024-02-20T22:44:22.807Z 15 300-319 54 54 54 100 100 100 F F F
64132024-02-20T22:44:22.807Z 16 320-339 54 54 54 100 100 100 F F F
64142024-02-20T22:44:22.807Z 17 340-359 51 51 51 97 97 97 F F F
64152024-02-20T22:44:22.807Z 18 360-379 55 55 55 101 101 101 F F F
64162024-02-20T22:44:22.807Z 19 380-399 55 55 55 101 101 101 F F F
64172024-02-20T22:44:22.808Z 20 400-419 55 55 55 101 101 101 F F F
64182024-02-20T22:44:22.808Z 21 420-439 55 55 55 101 101 101 F F F
64192024-02-20T22:44:22.808Z 22 440-459 58 55 58 102 101 102 F F F <---
64202024-02-20T22:44:22.808Z 23 460-479 58 55 58 105 101 105 F F F <---
64212024-02-20T22:44:22.808Z 24 480-499 53 53 53 99 99 99 T F T <---
64222024-02-20T22:44:22.808Z 25 500-519 52 52 52 98 98 98 F F F
64232024-02-20T22:44:22.808Z 26 520-539 58 53 58 105 99 105 T F T <---
64242024-02-20T22:44:22.808Z 27 540-559 58 52 58 105 98 105 T F T <---
64252024-02-20T22:44:22.808Z 28 560-579 53 53 53 99 99 99 F F F
64262024-02-20T22:44:22.808Z 29 580-599 55 55 55 101 101 101 F F F
64272024-02-20T22:44:22.808ZMax gen: 58, Max flush: 105
64282024-02-20T22:44:22.808ZError: Difference in extent metadata found!
64292024-02-20T22:44:22.809ZA Difference in extent metadata is expected here
64302024-02-20T22:44:22.809ZOn loop 29
64312024-02-20T22:44:22.809Z
64322024-02-20T22:44:22.809ZVerifying data now
64332024-02-20T22:44:27.447ZNo change after verify
64342024-02-20T22:44:27.447ZLoop: 29 Downstairs dump after verify (and repair):
64352024-02-20T22:44:27.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64362024-02-20T22:44:27.462Z 0 000-019 55 55 55 101 101 101 F F F
64372024-02-20T22:44:27.462Z 1 020-039 58 58 58 105 105 105 F F F
64382024-02-20T22:44:27.462Z 2 040-059 58 58 58 105 105 105 F F F
64392024-02-20T22:44:27.462Z 3 060-079 59 59 59 106 106 106 F F F
64402024-02-20T22:44:27.462Z 4 080-099 55 55 55 101 101 101 F F F
64412024-02-20T22:44:27.463Z 5 100-119 58 58 58 105 105 105 F F F
64422024-02-20T22:44:27.467Z 6 120-139 58 58 58 105 105 105 F F F
64432024-02-20T22:44:27.467Z 7 140-159 55 55 55 101 101 101 F F F
64442024-02-20T22:44:27.468Z 8 160-179 54 54 54 100 100 100 F F F
64452024-02-20T22:44:27.468Z 9 180-199 58 58 58 105 105 105 F F F
64462024-02-20T22:44:27.468Z 10 200-219 58 58 58 105 105 105 F F F
64472024-02-20T22:44:27.468Z 11 220-239 55 55 55 101 101 101 F F F
64482024-02-20T22:44:27.468Z 12 240-259 54 54 54 100 100 100 F F F
64492024-02-20T22:44:27.468Z 13 260-279 50 50 50 96 96 96 F F F
64502024-02-20T22:44:27.468Z 14 280-299 58 58 58 104 104 104 F F F
64512024-02-20T22:44:27.468Z 15 300-319 54 54 54 100 100 100 F F F
64522024-02-20T22:44:27.468Z 16 320-339 54 54 54 100 100 100 F F F
64532024-02-20T22:44:27.468Z 17 340-359 51 51 51 97 97 97 F F F
64542024-02-20T22:44:27.468Z 18 360-379 55 55 55 101 101 101 F F F
64552024-02-20T22:44:27.468Z 19 380-399 55 55 55 101 101 101 F F F
64562024-02-20T22:44:27.468Z 20 400-419 55 55 55 101 101 101 F F F
64572024-02-20T22:44:27.468Z 21 420-439 55 55 55 101 101 101 F F F
64582024-02-20T22:44:27.468Z 22 440-459 58 58 58 102 102 102 F F F
64592024-02-20T22:44:27.468Z 23 460-479 58 58 58 105 105 105 F F F
64602024-02-20T22:44:27.468Z 24 480-499 59 59 59 106 106 106 F F F
64612024-02-20T22:44:27.468Z 25 500-519 52 52 52 98 98 98 F F F
64622024-02-20T22:44:27.469Z 26 520-539 59 59 59 106 106 106 F F F
64632024-02-20T22:44:27.469Z 27 540-559 59 59 59 106 106 106 F F F
64642024-02-20T22:44:27.469Z 28 560-579 53 53 53 99 99 99 F F F
64652024-02-20T22:44:27.469Z 29 580-599 55 55 55 101 101 101 F F F
64662024-02-20T22:44:27.469ZMax gen: 59, Max flush: 106
64672024-02-20T22:44:27.469Z
64682024-02-20T22:44:27.469ZBegin loop 30
64692024-02-20T22:44:27.469ZDownstairs to restart: 0
64702024-02-20T22:44:27.487Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014266304Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378}
64712024-02-20T22:44:27.498Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014591612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378}
64722024-02-20T22:44:27.498Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014627441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378}
64732024-02-20T22:44:27.498Z{"msg":"Crucible a832be4d-042d-4d1f-a924-024eb97fb00f has session id: 639749ff-eb31-445e-af1b-96814d1bb5ab","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.01465725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64742024-02-20T22:44:27.498Z{"msg":"Upstairs opts: Upstairs UUID: a832be4d-042d-4d1f-a924-024eb97fb00f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014701038Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378Crucible runtime is spawned
64752024-02-20T22:44:27.498Z,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64762024-02-20T22:44:27.498Z{"msg":"Crucible stats registered with UUID: a832be4d-042d-4d1f-a924-024eb97fb00f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014784205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64772024-02-20T22:44:27.498Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014892362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378}
64782024-02-20T22:44:27.498Z{"msg":"a832be4d-042d-4d1f-a924-024eb97fb00f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.014974159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64792024-02-20T22:44:27.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015018157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64802024-02-20T22:44:27.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015037616Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64812024-02-20T22:44:27.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015063805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64822024-02-20T22:44:27.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015079245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64832024-02-20T22:44:27.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015093254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64842024-02-20T22:44:27.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015123853Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64852024-02-20T22:44:27.498Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015149372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"io task","client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64862024-02-20T22:44:27.498Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015186181Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"io task","client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64872024-02-20T22:44:27.498Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.01521234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"io task","client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64882024-02-20T22:44:27.498Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015301507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"io task","client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64892024-02-20T22:44:27.499Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.01548877Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"io task","client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64902024-02-20T22:44:27.499Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.015527299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"io task","client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64912024-02-20T22:44:27.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016068749Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64922024-02-20T22:44:27.499Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016111508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64932024-02-20T22:44:27.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016128467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64942024-02-20T22:44:27.499Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016142437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64952024-02-20T22:44:27.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016155946Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64962024-02-20T22:44:27.499Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016170826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64972024-02-20T22:44:27.499Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016683867Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64982024-02-20T22:44:27.499Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016723106Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
64992024-02-20T22:44:27.499Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.016949718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65002024-02-20T22:44:27.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017198969Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65012024-02-20T22:44:27.499Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017277226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65022024-02-20T22:44:27.499Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017317375Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65032024-02-20T22:44:27.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017333694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65042024-02-20T22:44:27.499Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017354573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65052024-02-20T22:44:27.499Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017383902Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65062024-02-20T22:44:27.500Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017400192Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65072024-02-20T22:44:27.500Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017430671Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65082024-02-20T22:44:27.500Z{"msg":"[0]R flush_numbers[0..12]: [101, 105, 105, 106, 101, 105, 105, 101, 100, 105, 105, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.01745263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65092024-02-20T22:44:27.500Z{"msg":"[0]R generation[0..12]: [55, 58, 58, 59, 55, 58, 58, 55, 54, 58, 58, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017484109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65102024-02-20T22:44:27.500Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017504798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65112024-02-20T22:44:27.500Z{"msg":"[1]R flush_numbers[0..12]: [101, 105, 105, 106, 101, 105, 105, 101, 100, 105, 105, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017534887Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65122024-02-20T22:44:27.500Z{"msg":"[1]R generation[0..12]: [55, 58, 58, 59, 55, 58, 58, 55, 54, 58, 58, 55]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
65132024-02-20T22:44:27.500Zlevel":30,"time":"2024-02-20T22:44:27.017570126Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65142024-02-20T22:44:27.500Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017602954Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65152024-02-20T22:44:27.500Z{"msg":"[2]R flush_numbers[0..12]: [101, 105, 105, 106, 101, 105, 105, 101, 100, 105, 105, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017635443Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65162024-02-20T22:44:27.500Z{"msg":"[2]R generation[0..12]: [55, 58, 58, 59, 55, 58, 58, 55, 54, 58, 58, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017668562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65172024-02-20T22:44:27.500Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017686662Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65182024-02-20T22:44:27.500Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.01771655Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65192024-02-20T22:44:27.500Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.01773229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65202024-02-20T22:44:27.500Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017762869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65212024-02-20T22:44:27.500Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017781458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65222024-02-20T22:44:27.500Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017811947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65232024-02-20T22:44:27.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017826226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65242024-02-20T22:44:27.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017857115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65252024-02-20T22:44:27.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017886704Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65262024-02-20T22:44:27.500Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017910514Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65272024-02-20T22:44:27.500Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65282024-02-20T22:44:27.501Z,"level":30,"time":"2024-02-20T22:44:27.017940862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65292024-02-20T22:44:27.501Z{"msg":"a832be4d-042d-4d1f-a924-024eb97fb00f is now active with session: 639749ff-eb31-445e-af1b-96814d1bb5ab","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017958162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65302024-02-20T22:44:27.501Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.017986501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378,"session_id":"639749ff-eb31-445e-af1b-96814d1bb5ab"}
65312024-02-20T22:44:27.501Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:27.018048729Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378}
65322024-02-20T22:44:27.501ZLoading write count information from file "/tmp/test_repair_verify.data"
65332024-02-20T22:44:27.501ZRun Repair workload
65342024-02-20T22:44:27.501Z01/30 Write block 154 len 28672 data: 4 4 5 6 6 6 5
65352024-02-20T22:44:27.501Z02/30 Read block 377 len 4096
65362024-02-20T22:44:27.501Z03/30 Write block 124 len 36864 data: 4 4 2 5 5 5 4 5 5
65372024-02-20T22:44:27.501Z04/30 Read block 6 len 40960
65382024-02-20T22:44:27.505Z05/30 Write block 563 len 32768 data: 4 6 4 5 5 5 4 5
65392024-02-20T22:44:27.505Z06/30 Write block 248 len 4096 data: 4
65402024-02-20T22:44:27.505Z07/30 Write block 317 len 4096 data: 8
65412024-02-20T22:44:27.505Z08/30 Write block 360 len 20480 data: 9 9 10 13 11
65422024-02-20T22:44:27.506Z09/30 Write block 140 len 28672 data: 6 5 6 7 6 5 3
65432024-02-20T22:44:27.506Z10/30 Write block 370 len 36864 data: 8 9 8 5 5 5 6 7 6
65442024-02-20T22:44:27.506Z11/30 Read block 213 len 24576
65452024-02-20T22:44:27.506Z12/30 Write block 210 len 12288 data: 4 4 4
65462024-02-20T22:44:27.506Z13/30 Write block 451 len 24576 data: 5 8 7 8 9 11
65472024-02-20T22:44:27.506Z14/30 Write block 502 len 16384 data: 4 3 4 4
65482024-02-20T22:44:27.506Z15/30 Write block 468 len 20480 data: 9 10 7 6 6
65492024-02-20T22:44:27.506Z16/30 Write block 99 len 36864 data: 5 6 4 3 2 3 3 3 3
65502024-02-20T22:44:27.506Z17/30 Read block 209 len 24576
65512024-02-20T22:44:27.506Z18/30 Read block 516 len 36864
65522024-02-20T22:44:27.506Z19/30 Write block 583 len 16384 data: 5 3 5 6
65532024-02-20T22:44:27.506Z20/30 Write block 422 len 8192 data: 5 5
65542024-02-20T22:44:27.516Z21/30 Write block 373 len 40960 data: 6 6 6 7 8 7 4 4 2 4
65552024-02-20T22:44:27.516Z22/30 Read block 525 len 36864
65562024-02-20T22:44:27.516Z23/30 Write block 438 len 20480 data: 5 7 8 8 9
65572024-02-20T22:44:27.516Z24/30 Write block 472 len 28672 data: 7 7 8 8 7 9 8
65582024-02-20T22:44:27.516Z25/30 Flush
65592024-02-20T22:44:27.522Z26/30 Write block 133 len 4096 data: 6
65602024-02-20T22:44:27.523Z27/30 Write block 228 len 40960 data: 6 7 9 7 8 10 10 6 6 6
65612024-02-20T22:44:27.523Z28/30 Read block 294 len 28672
65622024-02-20T22:44:27.523Z29/30 Write block 238 len 8192 data: 4 4
65632024-02-20T22:44:27.523Z30/30 Write block 302 len 40960 data: 5 5 4 4 6 6 5 6 6 6
65642024-02-20T22:44:27.523Z----------------------------------------------------------------
65652024-02-20T22:44:27.523Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65662024-02-20T22:44:27.523ZStates: Active Active Active
65672024-02-20T22:44:27.523ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65682024-02-20T22:44:27.523Z 1 Acked 1000 Write 7 Done Done Done false
65692024-02-20T22:44:27.523Z 2 Acked 1001 Read 1 Done Done Done false
65702024-02-20T22:44:27.524Z 3 Acked 1002 Write 9 Sent Done Done false
65712024-02-20T22:44:27.524Z 4 Acked 1003 Read 10 Sent Done Done false
65722024-02-20T22:44:27.524Z 5 Acked 1004 Write 8 Sent Done Done false
65732024-02-20T22:44:27.524Z 6 Acked 1005 Write 1 Sent Done Done false
65742024-02-20T22:44:27.524Z 7 Acked 1006 Write 1 Sent Done Done false
65752024-02-20T22:44:27.524Z 8 Acked 1007 Write 5 Sent Done Done false
65762024-02-20T22:44:27.524Z 9 Acked 1008 Write 7 Sent Done Done false
65772024-02-20T22:44:27.524Z 10 Acked 1009 Write 9 Sent Done Done false
65782024-02-20T22:44:27.524Z 11 Acked 1010 Read 6 Sent Done Done false
65792024-02-20T22:44:27.524Z 12 Acked 1011 Write 3 Sent Done Done false
65802024-02-20T22:44:27.524Z 13 Acked 1012 Write 6 Sent Done Done false
65812024-02-20T22:44:27.524Z 14 Acked 1013 Write 4 Sent Done Done false
65822024-02-20T22:44:27.524Z 15 Acked 1014 Write 5 Sent Done Done false
65832024-02-20T22:44:27.525Z 16 Acked 1015 Write 9 Sent Done Done false
65842024-02-20T22:44:27.525Z 17 Acked 1016 Read 6 Sent Done Done false
65852024-02-20T22:44:27.525Z 18 Acked 1017 Read 9 Sent Done Done false
65862024-02-20T22:44:27.525Z 19 Acked 1018 Write 4 Sent Done Done false
65872024-02-20T22:44:27.525Z 20 Acked 1019 Write 2 Sent Done Done false
65882024-02-20T22:44:27.525Z 21 Acked 1020 Write 10 Sent Done Done false
65892024-02-20T22:44:27.525Z 22 Acked 1021 Read 9 Sent Done Done false
65902024-02-20T22:44:27.525Z 23 Acked 1022 Write 5 Sent Done Done false
65912024-02-20T22:44:27.525Z 24 Acked 1023 Write 7 Sent Done Done false
65922024-02-20T22:44:27.525Z 25 Acked 1024 Flush 0 Sent Done Done false
65932024-02-20T22:44:27.525Z 26 Acked 1025 Write 1 Sent Done Done false
65942024-02-20T22:44:27.525Z 27 Acked 1026 Write 10 Sent Done Done false
65952024-02-20T22:44:27.526Z 28 Acked 1027 Read 7 Sent Done Done false
65962024-02-20T22:44:27.526Z 29 Acked 1028 Write 2 Sent Done Done false
65972024-02-20T22:44:27.526Z 30 Acked 1029 Write 10 Sent Sent Sent false
65982024-02-20T22:44:27.526Z STATES DS:0 DS:1 DS:2 TOTAL
65992024-02-20T22:44:27.526Z New 0 0 0 0
66002024-02-20T22:44:27.526Z Sent 28 1 1 30
66012024-02-20T22:44:27.526Z Done 2 29 29 60
66022024-02-20T22:44:27.526Z Skipped 0 0 0 0
66032024-02-20T22:44:27.526Z Error 0 0 0 0
66042024-02-20T22:44:27.526ZLast Flush: 0 1024 1024
66052024-02-20T22:44:27.527ZDownstairs last five completed:
66062024-02-20T22:44:27.527ZUpstairs last five completed: 30 29 28 27 26
66072024-02-20T22:44:27.528Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:27.050900943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1378}
66082024-02-20T22:44:27.534ZWrote out file "/tmp/test_repair_verify.data"
66092024-02-20T22:44:27.534Z
66102024-02-20T22:44:27.534ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66112024-02-20T22:44:27.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66122024-02-20T22:44:27.544Z 0 000-019 55 55 55 101 101 101 F F F
66132024-02-20T22:44:27.544Z 1 020-039 58 58 58 105 105 105 F F F
66142024-02-20T22:44:27.544Z 2 040-059 58 58 58 105 105 105 F F F
66152024-02-20T22:44:27.545Z 3 060-079 59 59 59 106 106 106 F F F
66162024-02-20T22:44:27.545Z 4 080-099 55 60 60 101 107 107 F F F <---
66172024-02-20T22:44:27.545Z 5 100-119 58 60 60 105 107 107 F F F <---
66182024-02-20T22:44:27.545Z 6 120-139 58 60 60 105 107 107 F T T <---
66192024-02-20T22:44:27.545Z 7 140-159 55 60 60 101 107 107 T F F <---
66202024-02-20T22:44:27.545Z 8 160-179 54 60 60 100 107 107 T F F <---
66212024-02-20T22:44:27.545Z 9 180-199 58 58 58 105 105 105 F F F
66222024-02-20T22:44:27.550Z 10 200-219 58 60 60 105 107 107 F F F <---
66232024-02-20T22:44:27.550Z 11 220-239 55 55 55 101 101 101 F T T <---
66242024-02-20T22:44:27.550Z 12 240-259 54 60 60 100 107 107 F F F <---
66252024-02-20T22:44:27.550Z 13 260-279 50 50 50 96 96 96 F F F
66262024-02-20T22:44:27.550Z 14 280-299 58 58 58 104 104 104 F F F
66272024-02-20T22:44:27.550Z 15 300-319 54 60 60 100 107 107 F T T <---
66282024-02-20T22:44:27.550Z 16 320-339 54 54 54 100 100 100 F F F
66292024-02-20T22:44:27.550Z 17 340-359 51 51 51 97 97 97 F F F
66302024-02-20T22:44:27.550Z 18 360-379 55 60 60 101 107 107 F F F <---
66312024-02-20T22:44:27.550Z 19 380-399 55 60 60 101 107 107 F F F <---
66322024-02-20T22:44:27.551Z 20 400-419 55 55 55 101 101 101 F F F
66332024-02-20T22:44:27.551Z 21 420-439 55 60 60 101 107 107 F F F <---
66342024-02-20T22:44:27.551Z 22 440-459 58 60 60 102 107 107 F F F <---
66352024-02-20T22:44:27.551Z 23 460-479 58 60 60 105 107 107 F F F <---
66362024-02-20T22:44:27.551Z 24 480-499 59 59 59 106 106 106 F F F
66372024-02-20T22:44:27.551Z 25 500-519 52 60 60 98 107 107 F F F <---
66382024-02-20T22:44:27.552Z 26 520-539 59 59 59 106 106 106 F F F
66392024-02-20T22:44:27.552Z 27 540-559 59 59 59 106 106 106 F F F
66402024-02-20T22:44:27.552Z 28 560-579 53 60 60 99 107 107 F F F <---
66412024-02-20T22:44:27.552Z 29 580-599 55 60 60 101 107 107 F F F <---
66422024-02-20T22:44:27.552ZMax gen: 60, Max flush: 107
66432024-02-20T22:44:27.552ZError: Difference in extent metadata found!
66442024-02-20T22:44:27.552ZA Difference in extent metadata is expected here
66452024-02-20T22:44:27.552ZOn loop 30
66462024-02-20T22:44:27.552Z
66472024-02-20T22:44:27.558ZVerifying data now
66482024-02-20T22:44:33.325ZNo change after verify
66492024-02-20T22:44:33.325ZLoop: 30 Downstairs dump after verify (and repair):
66502024-02-20T22:44:33.337ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66512024-02-20T22:44:33.337Z 0 000-019 55 55 55 101 101 101 F F F
66522024-02-20T22:44:33.337Z 1 020-039 58 58 58 105 105 105 F F F
66532024-02-20T22:44:33.337Z 2 040-059 58 58 58 105 105 105 F F F
66542024-02-20T22:44:33.338Z 3 060-079 59 59 59 106 106 106 F F F
66552024-02-20T22:44:33.338Z 4 080-099 60 60 60 107 107 107 F F F
66562024-02-20T22:44:33.338Z 5 100-119 60 60 60 107 107 107 F F F
66572024-02-20T22:44:33.338Z 6 120-139 61 61 61 108 108 108 F F F
66582024-02-20T22:44:33.338Z 7 140-159 60 60 60 107 107 107 F F F
66592024-02-20T22:44:33.338Z 8 160-179 60 60 60 107 107 107 F F F
66602024-02-20T22:44:33.338Z 9 180-199 58 58 58 105 105 105 F F F
66612024-02-20T22:44:33.338Z 10 200-219 60 60 60 107 107 107 F F F
66622024-02-20T22:44:33.355Z 11 220-239 61 61 61 108 108 108 F F F
66632024-02-20T22:44:33.355Z 12 240-259 60 60 60 107 107 107 F F F
66642024-02-20T22:44:33.356Z 13 260-279 50 50 50 96 96 96 F F F
66652024-02-20T22:44:33.356Z 14 280-299 58 58 58 104 104 104 F F F
66662024-02-20T22:44:33.356Z 15 300-319 61 61 61 108 108 108 F F F
66672024-02-20T22:44:33.356Z 16 320-339 54 54 54 100 100 100 F F F
66682024-02-20T22:44:33.356Z 17 340-359 51 51 51 97 97 97 F F F
66692024-02-20T22:44:33.356Z 18 360-379 60 60 60 107 107 107 F F F
66702024-02-20T22:44:33.356Z 19 380-399 60 60 60 107 107 107 F F F
66712024-02-20T22:44:33.356Z 20 400-419 55 55 55 101 101 101 F F F
66722024-02-20T22:44:33.356Z 21 420-439 60 60 60 107 107 107 F F F
66732024-02-20T22:44:33.356Z 22 440-459 60 60 60 107 107 107 F F F
66742024-02-20T22:44:33.356Z 23 460-479 60 60 60 107 107 107 F F F
66752024-02-20T22:44:33.356Z 24 480-499 59 59 59 106 106 106 F F F
66762024-02-20T22:44:33.356Z 25 500-519 60 60 60 107 107 107 F F F
66772024-02-20T22:44:33.357Z 26 520-539 59 59 59 106 106 106 F F F
66782024-02-20T22:44:33.357Z 27 540-559 59 59 59 106 106 106 F F F
66792024-02-20T22:44:33.357Z 28 560-579 60 60 60 107 107 107 F F F
66802024-02-20T22:44:33.357Z 29 580-599 60 60 60 107 107 107 F F F
66812024-02-20T22:44:33.357ZMax gen: 61, Max flush: 108
66822024-02-20T22:44:33.357Z
66832024-02-20T22:44:33.357ZBegin loop 31
66842024-02-20T22:44:33.357ZDownstairs to restart: 1
66852024-02-20T22:44:33.381Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.889412404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386}
66862024-02-20T22:44:33.381Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.889688324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386}
66872024-02-20T22:44:33.381Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.889751682Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386}
66882024-02-20T22:44:33.381Z{"msg":"Crucible 7408ee89-bc3d-40ba-be60-040d79ba8676 has session id: 7a2cd1d7-836b-4501-a37c-7b341d869cd3","v":0,"name":"crucible","level":30Crucible runtime is spawned
66892024-02-20T22:44:33.381Z,"time":"2024-02-20T22:44:32.88981292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66902024-02-20T22:44:33.381Z{"msg":"Upstairs opts: Upstairs UUID: 7408ee89-bc3d-40ba-be60-040d79ba8676, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.889887677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66912024-02-20T22:44:33.382Z{"msg":"Crucible stats registered with UUID: 7408ee89-bc3d-40ba-be60-040d79ba8676","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.889926186Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66922024-02-20T22:44:33.382Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.889963114Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386}
66932024-02-20T22:44:33.382Z{"msg":"7408ee89-bc3d-40ba-be60-040d79ba8676 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890056651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66942024-02-20T22:44:33.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89008346Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66952024-02-20T22:44:33.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890099979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66962024-02-20T22:44:33.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890120739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66972024-02-20T22:44:33.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890135898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66982024-02-20T22:44:33.382Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890151118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
66992024-02-20T22:44:33.382Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890173157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67002024-02-20T22:44:33.382Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890188166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"io task","client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67012024-02-20T22:44:33.382Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890218585Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"io task","client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67022024-02-20T22:44:33.382Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890299752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"io task","client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67032024-02-20T22:44:33.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890408368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"io task","client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67042024-02-20T22:44:33.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890475926Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"io task","client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67052024-02-20T22:44:33.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.890532684Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"io task","client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67062024-02-20T22:44:33.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891122043Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67072024-02-20T22:44:33.382Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891159852Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67082024-02-20T22:44:33.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891177171Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67092024-02-20T22:44:33.382Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89119236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67102024-02-20T22:44:33.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891221569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67112024-02-20T22:44:33.382Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891252888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67122024-02-20T22:44:33.383Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891849207Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67132024-02-20T22:44:33.383Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891896725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67142024-02-20T22:44:33.383Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.891933154Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67152024-02-20T22:44:33.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89231161Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67162024-02-20T22:44:33.383Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892362278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67172024-02-20T22:44:33.383Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892381098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67182024-02-20T22:44:33.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892405147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67192024-02-20T22:44:33.383Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892432876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67202024-02-20T22:44:33.383Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892483884Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67212024-02-20T22:44:33.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892530232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67222024-02-20T22:44:33.383Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892569571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67232024-02-20T22:44:33.383Z{"msg":"[0]R flush_numbers[0..12]: [101, 105, 105, 106, 107, 107, 108, 107, 107, 105, 107, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89259455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67242024-02-20T22:44:33.383Z{"msg":"[0]R generation[0..12]: [55, 58, 58, 59, 60, 60, 61, 60, 60, 58, 60, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892630489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67252024-02-20T22:44:33.383Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892666668Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67262024-02-20T22:44:33.383Z{"msg":"[1]R flush_numbers[0..12]: [101, 105, 105, 106, 107, 107, 108, 107, 107, 105, 107, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892702746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67272024-02-20T22:44:33.383Z{"msg":"[1]R generation[0..12]: [55, 58, 58, 59, 60, 60, 61, 60, 60, 58, 60, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892739445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67282024-02-20T22:44:33.383ZWait for a query_work_queue command to finish before sending IO
67292024-02-20T22:44:33.383Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892783974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67302024-02-20T22:44:33.383Z{"msg":"[2]R flush_numbers[0..12]: [101, 105, 105, 106, 107, 107, 108, 107, 107, 105, 107, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892815802Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67312024-02-20T22:44:33.383Z{"msg":"[2]R generation[0..12]: [55, 58, 58, 59, 60, 60, 61, 60, 60, 58, 60, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892855481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67322024-02-20T22:44:33.383Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89288584Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67332024-02-20T22:44:33.383Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892918329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67342024-02-20T22:44:33.383Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892939188Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67352024-02-20T22:44:33.383Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892967477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67362024-02-20T22:44:33.383Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.892985496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67372024-02-20T22:44:33.384Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893009825Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67382024-02-20T22:44:33.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893023705Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67392024-02-20T22:44:33.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893054544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67402024-02-20T22:44:33.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893083623Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67412024-02-20T22:44:33.384Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893112472Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67422024-02-20T22:44:33.384Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893128441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67432024-02-20T22:44:33.384Z{"msg":"7408ee89-bc3d-40ba-be60-040d79ba8676 is now active with session: 7a2cd1d7-836b-4501-a37c-7b341d869cd3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89315278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67442024-02-20T22:44:33.384Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.89316646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386,"session_id":"7a2cd1d7-836b-4501-a37c-7b341d869cd3"}
67452024-02-20T22:44:33.384Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:32.893184539Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386}
67462024-02-20T22:44:33.384ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67472024-02-20T22:44:33.384ZLoading write count information from file "/tmp/test_repair_verify.data"
67482024-02-20T22:44:33.384ZRun Repair workload
67492024-02-20T22:44:33.384Z01/30 Flush
67502024-02-20T22:44:33.384Z02/30 Write block 377 len 4096 data: 9
67512024-02-20T22:44:33.384Z03/30 Read block 57 len 12288
67522024-02-20T22:44:33.384Z04/30 Write block 497 len 12288 data: 11 10 8
67532024-02-20T22:44:33.384Z05/30 Read block 52 len 20480
67542024-02-20T22:44:33.384Z06/30 Read block 132 len 36864
67552024-02-20T22:44:33.384Z07/30 Read block 36 len 40960
67562024-02-20T22:44:33.384Z08/30 Write block 405 len 40960 data: 7 8 8 6 9 9 9 9 8 8
67572024-02-20T22:44:33.384Z09/30 Write block 209 len 20480 data: 5 5 5 5 3
67582024-02-20T22:44:33.384Z10/30 Read block 284 len 4096
67592024-02-20T22:44:33.385Z11/30 Write block 481 len 24576 data: 10 9 9 8 6 5
67602024-02-20T22:44:33.385Z12/30 Write block 195 len 20480 data: 6 6 7 6 7
67612024-02-20T22:44:33.385Z13/30 Read block 175 len 28672
67622024-02-20T22:44:33.391Z14/30 Read block 225 len 20480
67632024-02-20T22:44:33.391Z15/30 Write block 434 len 32768 data: 3 3 3 4 6 8 9 9
67642024-02-20T22:44:33.391Z16/30 Read block 106 len 40960
67652024-02-20T22:44:33.391Z17/30 Write block 387 len 28672 data: 3 6 7 7 6 8 8
67662024-02-20T22:44:33.391Z18/30 Read block 512 len 16384
67672024-02-20T22:44:33.392Z19/30 Read block 181 len 20480
67682024-02-20T22:44:33.392Z20/30 Read block 409 len 16384
67692024-02-20T22:44:33.392Z21/30 Write block 511 len 28672 data: 9 5 5 5 6 5 4
67702024-02-20T22:44:33.392Z22/30 Read block 126 len 24576
67712024-02-20T22:44:33.392Z23/30 Flush
67722024-02-20T22:44:33.392Z24/30 Write block 403 len 8192 data: 7 7
67732024-02-20T22:44:33.392Z25/30 Read block 130 len 28672
67742024-02-20T22:44:33.392Z26/30 Write block 477 len 8192 data: 10 9
67752024-02-20T22:44:33.398Z27/30 Write block 428 len 32768 data: 3 3 3 3 2 2 4 4
67762024-02-20T22:44:33.398Z28/30 Write block 231 len 28672 data: 8 9 11 11 7 7 7
67772024-02-20T22:44:33.398Z29/30 Read block 392 len 36864
67782024-02-20T22:44:33.398Z30/30 Write block 247 len 24576 data: 5 5 4 4 3 3
67792024-02-20T22:44:33.398Z----------------------------------------------------------------
67802024-02-20T22:44:33.398Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:29
67812024-02-20T22:44:33.398ZStates: Active Active Active
67822024-02-20T22:44:33.398ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67832024-02-20T22:44:33.398Z 2 Acked 1001 Write 1 Done Sent Done false
67842024-02-20T22:44:33.398Z 3 Acked 1002 Read 3 Done Sent Done false
67852024-02-20T22:44:33.398Z 4 Acked 1003 Write 3 Done Sent Done false
67862024-02-20T22:44:33.399Z 5 Acked 1004 Read 5 Done Sent Done false
67872024-02-20T22:44:33.399Z 6 Acked 1005 Read 9 Done Sent Done false
67882024-02-20T22:44:33.399Z 7 Acked 1006 Read 10 Done Sent Done false
67892024-02-20T22:44:33.399Z 8 Acked 1007 Write 10 Done Sent Done false
67902024-02-20T22:44:33.399Z 9 Acked 1008 Write 5 Done Sent Done false
67912024-02-20T22:44:33.399Z 10 Acked 1009 Read 1 Done Sent Done false
67922024-02-20T22:44:33.399Z 11 Acked 1010 Write 6 Done Sent Done false
67932024-02-20T22:44:33.399Z 12 Acked 1011 Write 5 Done Sent Done false
67942024-02-20T22:44:33.399Z 13 Acked 1012 Read 7 Done Sent Done false
67952024-02-20T22:44:33.399Z 14 Acked 1013 Read 5 Done Sent Done false
67962024-02-20T22:44:33.399Z 15 Acked 1014 Write 8 Done Sent Done false
67972024-02-20T22:44:33.399Z 16 Acked 1015 Read 10 Done Sent Done false
67982024-02-20T22:44:33.399Z 17 Acked 1016 Write 7 Done Sent Done false
67992024-02-20T22:44:33.399Z 18 Acked 1017 Read 4 Done Sent Done false
68002024-02-20T22:44:33.399Z 19 Acked 1018 Read 5 Done Sent Done false
68012024-02-20T22:44:33.399Z 20 Acked 1019 Read 4 Done Sent Done false
68022024-02-20T22:44:33.399Z 21 Acked 1020 Write 7 Done Sent Done false
68032024-02-20T22:44:33.399Z 22 Acked 1021 Read 6 Done Sent Done false
68042024-02-20T22:44:33.399Z 23 Acked 1022 Flush 0 Done Sent Done false
68052024-02-20T22:44:33.399Z 24 Acked 1023 Write 2 Done Sent Done false
68062024-02-20T22:44:33.399Z 25 Acked 1024 Read 7 Done Sent Done false
68072024-02-20T22:44:33.399Z 26 Acked 1025 Write 2 Done Sent Done false
68082024-02-20T22:44:33.400Z 27 Acked 1026 Write 8 Done Sent Done false
68092024-02-20T22:44:33.400Z 28 Acked 1027 Write 7 Done Sent Done false
68102024-02-20T22:44:33.400Z 29 Acked 1028 Read 9 Done Sent Done false
68112024-02-20T22:44:33.400Z 30 Acked 1029 Write 6 Sent Sent Sent false
68122024-02-20T22:44:33.400Z STATES DS:0 DS:1 DS:2 TOTAL
68132024-02-20T22:44:33.400Z New 0 0 0 0
68142024-02-20T22:44:33.400Z Sent 1 29 1 31
68152024-02-20T22:44:33.400Z Done 28 0 28 56
68162024-02-20T22:44:33.400Z Skipped 0 0 0 0
68172024-02-20T22:44:33.400Z Error 0 0 0 0
68182024-02-20T22:44:33.400ZLast Flush: 1022 1000 1022
68192024-02-20T22:44:33.400ZDownstairs last five completed: 1000
68202024-02-20T22:44:33.400ZUpstairs last five completed: 30 29 28 27 26
68212024-02-20T22:44:33.400Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:32.924343914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1386}
68222024-02-20T22:44:33.400ZWrote out file "/tmp/test_repair_verify.data"
68232024-02-20T22:44:33.424Z
68242024-02-20T22:44:33.424ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68252024-02-20T22:44:33.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68262024-02-20T22:44:33.424Z 0 000-019 55 55 55 101 101 101 F F F
68272024-02-20T22:44:33.424Z 1 020-039 58 58 58 105 105 105 F F F
68282024-02-20T22:44:33.424Z 2 040-059 58 58 58 105 105 105 F F F
68292024-02-20T22:44:33.424Z 3 060-079 59 59 59 106 106 106 F F F
68302024-02-20T22:44:33.424Z 4 080-099 60 60 60 107 107 107 F F F
68312024-02-20T22:44:33.424Z 5 100-119 60 60 60 107 107 107 F F F
68322024-02-20T22:44:33.424Z 6 120-139 61 61 61 108 108 108 F F F
68332024-02-20T22:44:33.424Z 7 140-159 60 60 60 107 107 107 F F F
68342024-02-20T22:44:33.424Z 8 160-179 60 60 60 107 107 107 F F F
68352024-02-20T22:44:33.424Z 9 180-199 62 58 62 110 105 110 F F F <---
68362024-02-20T22:44:33.425Z 10 200-219 62 60 62 110 107 110 F F F <---
68372024-02-20T22:44:33.425Z 11 220-239 61 61 61 108 108 108 T F T <---
68382024-02-20T22:44:33.425Z 12 240-259 60 60 60 107 107 107 T F T <---
68392024-02-20T22:44:33.425Z 13 260-279 50 50 50 96 96 96 F F F
68402024-02-20T22:44:33.425Z 14 280-299 58 58 58 104 104 104 F F F
68412024-02-20T22:44:33.425Z 15 300-319 61 61 61 108 108 108 F F F
68422024-02-20T22:44:33.425Z 16 320-339 54 54 54 100 100 100 F F F
68432024-02-20T22:44:33.425Z 17 340-359 51 51 51 97 97 97 F F F
68442024-02-20T22:44:33.425Z 18 360-379 62 60 62 110 107 110 F F F <---
68452024-02-20T22:44:33.425Z 19 380-399 62 60 62 110 107 110 F F F <---
68462024-02-20T22:44:33.425Z 20 400-419 62 55 62 110 101 110 T F T <---
68472024-02-20T22:44:33.425Z 21 420-439 62 60 62 110 107 110 T F T <---
68482024-02-20T22:44:33.425Z 22 440-459 62 60 62 110 107 110 F F F <---
68492024-02-20T22:44:33.425Z 23 460-479 60 60 60 107 107 107 T F T <---
68502024-02-20T22:44:33.425Z 24 480-499 62 59 62 110 106 110 F F F <---
68512024-02-20T22:44:33.425Z 25 500-519 62 60 62 110 107 110 F F F <---
68522024-02-20T22:44:33.425Z 26 520-539 59 59 59 106 106 106 F F F
68532024-02-20T22:44:33.425Z 27 540-559 59 59 59 106 106 106 F F F
68542024-02-20T22:44:33.425Z 28 560-579 60 60 60 107 107 107 F F F
68552024-02-20T22:44:33.425Z 29 580-599 60 60 60 107 107 107 F F F
68562024-02-20T22:44:33.425ZMax gen: 62, Max flush: 110
68572024-02-20T22:44:33.425ZError: Difference in extent metadata found!
68582024-02-20T22:44:33.425ZA Difference in extent metadata is expected here
68592024-02-20T22:44:33.425ZOn loop 31
68602024-02-20T22:44:33.425Z
68612024-02-20T22:44:33.431ZVerifying data now
68622024-02-20T22:44:38.007ZNo change after verify
68632024-02-20T22:44:38.007ZLoop: 31 Downstairs dump after verify (and repair):
68642024-02-20T22:44:38.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68652024-02-20T22:44:38.025Z 0 000-019 55 55 55 101 101 101 F F F
68662024-02-20T22:44:38.025Z 1 020-039 58 58 58 105 105 105 F F F
68672024-02-20T22:44:38.025Z 2 040-059 58 58 58 105 105 105 F F F
68682024-02-20T22:44:38.025Z 3 060-079 59 59 59 106 106 106 F F F
68692024-02-20T22:44:38.025Z 4 080-099 60 60 60 107 107 107 F F F
68702024-02-20T22:44:38.025Z 5 100-119 60 60 60 107 107 107 F F F
68712024-02-20T22:44:38.025Z 6 120-139 61 61 61 108 108 108 F F F
68722024-02-20T22:44:38.029Z 7 140-159 60 60 60 107 107 107 F F F
68732024-02-20T22:44:38.029Z 8 160-179 60 60 60 107 107 107 F F F
68742024-02-20T22:44:38.030Z 9 180-199 62 62 62 110 110 110 F F F
68752024-02-20T22:44:38.030Z 10 200-219 62 62 62 110 110 110 F F F
68762024-02-20T22:44:38.030Z 11 220-239 63 63 63 111 111 111 F F F
68772024-02-20T22:44:38.030Z 12 240-259 63 63 63 111 111 111 F F F
68782024-02-20T22:44:38.030Z 13 260-279 50 50 50 96 96 96 F F F
68792024-02-20T22:44:38.030Z 14 280-299 58 58 58 104 104 104 F F F
68802024-02-20T22:44:38.030Z 15 300-319 61 61 61 108 108 108 F F F
68812024-02-20T22:44:38.030Z 16 320-339 54 54 54 100 100 100 F F F
68822024-02-20T22:44:38.030Z 17 340-359 51 51 51 97 97 97 F F F
68832024-02-20T22:44:38.030Z 18 360-379 62 62 62 110 110 110 F F F
68842024-02-20T22:44:38.030Z 19 380-399 62 62 62 110 110 110 F F F
68852024-02-20T22:44:38.030Z 20 400-419 63 63 63 111 111 111 F F F
68862024-02-20T22:44:38.030Z 21 420-439 63 63 63 111 111 111 F F F
68872024-02-20T22:44:38.030Z 22 440-459 62 62 62 110 110 110 F F F
68882024-02-20T22:44:38.030Z 23 460-479 63 63 63 111 111 111 F F F
68892024-02-20T22:44:38.030Z 24 480-499 62 62 62 110 110 110 F F F
68902024-02-20T22:44:38.030Z 25 500-519 62 62 62 110 110 110 F F F
68912024-02-20T22:44:38.030Z 26 520-539 59 59 59 106 106 106 F F F
68922024-02-20T22:44:38.030Z 27 540-559 59 59 59 106 106 106 F F F
68932024-02-20T22:44:38.030Z 28 560-579 60 60 60 107 107 107 F F F
68942024-02-20T22:44:38.030Z 29 580-599 60 60 60 107 107 107 F F F
68952024-02-20T22:44:38.030ZMax gen: 63, Max flush: 111
68962024-02-20T22:44:38.030Z
68972024-02-20T22:44:38.030ZBegin loop 32
68982024-02-20T22:44:38.031ZDownstairs to restart: 2
68992024-02-20T22:44:38.045Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.571867862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394}
69002024-02-20T22:44:38.055Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572127203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394}
69012024-02-20T22:44:38.055Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572164981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394}
69022024-02-20T22:44:38.055Z{"msg":"Crucible 703e5a58-0ace-4054-b8f0-8817e62f82e6 has session id: 12f35125-8198-4213-b4be-23f92cf5dd32","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572187271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69032024-02-20T22:44:38.056Z{"msg":"Upstairs opts: Upstairs UUID: 703e5a58-0ace-4054-b8f0-8817e62f82e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572228969Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69042024-02-20T22:44:38.056Z{"msg":"Crucible stats registered with UUID: 703e5a58-0ace-4054-b8f0-8817e62f82e6","v":0,"name":"crucibleCrucible runtime is spawned
69052024-02-20T22:44:38.056Z","level":30,"time":"2024-02-20T22:44:37.572273387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69062024-02-20T22:44:38.056Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572392813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394}
69072024-02-20T22:44:38.056Z{"msg":"703e5a58-0ace-4054-b8f0-8817e62f82e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572455111Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69082024-02-20T22:44:38.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572531648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69092024-02-20T22:44:38.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572563427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69102024-02-20T22:44:38.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572595256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69112024-02-20T22:44:38.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572621165Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69122024-02-20T22:44:38.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572661263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69132024-02-20T22:44:38.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572703672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69142024-02-20T22:44:38.056Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572730651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"io task","client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69152024-02-20T22:44:38.056Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572876696Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69162024-02-20T22:44:38.056Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572927934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"io task","client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69172024-02-20T22:44:38.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572961153Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"io task","client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69182024-02-20T22:44:38.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.572978082Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69192024-02-20T22:44:38.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.57303683Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"io task","client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69202024-02-20T22:44:38.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.573488184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69212024-02-20T22:44:38.056Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.573528323Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69222024-02-20T22:44:38.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.573554371Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69232024-02-20T22:44:38.056Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.573616989Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69242024-02-20T22:44:38.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.573732985Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69252024-02-20T22:44:38.056Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.573768674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69262024-02-20T22:44:38.056Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574047084Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69272024-02-20T22:44:38.057Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574091642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69282024-02-20T22:44:38.057Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574190029Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69292024-02-20T22:44:38.057Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574607024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69302024-02-20T22:44:38.057Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574659502Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69312024-02-20T22:44:38.057Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.57470535Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69322024-02-20T22:44:38.057Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574745029Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69332024-02-20T22:44:38.057Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574763738Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69342024-02-20T22:44:38.057Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574777668Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69352024-02-20T22:44:38.057Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574851805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69362024-02-20T22:44:38.057Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574879144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69372024-02-20T22:44:38.057Z{"msg":"[0]R flush_numbers[0..12]: [101, 105, 105, 106, 107, 107, 108, 107, 107, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574894603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69382024-02-20T22:44:38.057Z{"msg":"[0]R generation[0..12]: [55, 58, 58, 59, 60, 60, 61, 60, 60, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574909513Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69392024-02-20T22:44:38.057Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574931882Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69402024-02-20T22:44:38.057Z{"msg":"[1]R flush_numbers[0..12]: [101, 105, 105, 106, 107, 107, 108, 107, 107, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574946512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69412024-02-20T22:44:38.057Z{"msg":"[1]R generation[0..12]: [55, 58, 58, 59, 60, 60, 61, 60, 60, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574960371Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69422024-02-20T22:44:38.057Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.574975091Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69432024-02-20T22:44:38.057Z{"msg":"[2]R flush_numbers[0..12]: [101, 105, 105, 106, 107, 107, 108, 107, 107, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.57498896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69442024-02-20T22:44:38.057Z{"msg":"[2]R generation[0..12]: [55, 58, 58, 59, 60, 60, 61, 60, 60, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.57500979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69452024-02-20T22:44:38.057Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575024089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69462024-02-20T22:44:38.057Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575037978Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69472024-02-20T22:44:38.057ZWait for a query_work_queue command to finish before sending IO
69482024-02-20T22:44:38.057Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575067797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69492024-02-20T22:44:38.057Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575083497Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69502024-02-20T22:44:38.057Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575096806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69512024-02-20T22:44:38.057Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575110766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69522024-02-20T22:44:38.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575123675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69532024-02-20T22:44:38.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575137075Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69542024-02-20T22:44:38.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575158634Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69552024-02-20T22:44:38.058Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575172184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69562024-02-20T22:44:38.058Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575184423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69572024-02-20T22:44:38.058Z{"msg":"703e5a58-0ace-4054-b8f0-8817e62f82e6 is now active with session: 12f35125-8198-4213-b4be-23f92cf5dd32","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575197313Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69582024-02-20T22:44:38.058Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575216432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394,"session_id":"12f35125-8198-4213-b4be-23f92cf5dd32"}
69592024-02-20T22:44:38.058Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:37.575237911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394}
69602024-02-20T22:44:38.058ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69612024-02-20T22:44:38.058ZLoading write count information from file "/tmp/test_repair_verify.data"
69622024-02-20T22:44:38.058ZRun Repair workload
69632024-02-20T22:44:38.058Z01/30 Write block 263 len 4096 data: 3
69642024-02-20T22:44:38.058Z02/30 Read block 258 len 28672
69652024-02-20T22:44:38.058Z03/30 Write block 540 len 40960 data: 6 7 6 6 5 6 5 7 7 7
69662024-02-20T22:44:38.058Z04/30 Write block 304 len 36864 data: 5 5 7 7 6 7 7 7 6
69672024-02-20T22:44:38.065Z05/30 Read block 497 len 36864
69682024-02-20T22:44:38.065Z06/30 Read block 325 len 28672
69692024-02-20T22:44:38.065Z07/30 Read block 322 len 8192
69702024-02-20T22:44:38.065Z08/30 Write block 559 len 12288 data: 2 2 3
69712024-02-20T22:44:38.065Z09/30 Write block 64 len 24576 data: 3 4 4 4 5 6
69722024-02-20T22:44:38.065Z10/30 Write block 99 len 4096 data: 6
69732024-02-20T22:44:38.065Z11/30 Read block 22 len 32768
69742024-02-20T22:44:38.065Z12/30 Flush
69752024-02-20T22:44:38.065Z13/30 Write block 135 len 28672 data: 9 7 6 6 6 7 6
69762024-02-20T22:44:38.065Z14/30 Flush
69772024-02-20T22:44:38.071Z15/30 Write block 264 len 12288 data: 6 6 7
69782024-02-20T22:44:38.071Z16/30 Read block 570 len 4096
69792024-02-20T22:44:38.071Z17/30 Read block 453 len 16384
69802024-02-20T22:44:38.071Z18/30 Flush
69812024-02-20T22:44:38.071Z19/30 Write block 148 len 24576 data: 4 6 8 8 7 4
69822024-02-20T22:44:38.072Z20/30 Write block 291 len 12288 data: 5 4 3
69832024-02-20T22:44:38.072Z21/30 Read block 326 len 16384
69842024-02-20T22:44:38.072Z22/30 Read block 302 len 40960
69852024-02-20T22:44:38.072Z23/30 Write block 341 len 12288 data: 6 6 6
69862024-02-20T22:44:38.076Z24/30 Write block 579 len 24576 data: 7 7 5 6 6 4
69872024-02-20T22:44:38.076Z25/30 Read block 578 len 12288
69882024-02-20T22:44:38.076Z26/30 Read block 238 len 20480
69892024-02-20T22:44:38.076Z27/30 Write block 397 len 36864 data: 10 6 6 6 5 7 8 8 8
69902024-02-20T22:44:38.080Z28/30 Write block 115 len 32768 data: 6 7 8 4 6 8 7 7
69912024-02-20T22:44:38.080Z29/30 Write block 427 len 20480 data: 3 4 4 4 4
69922024-02-20T22:44:38.080Z30/30 Write block 35 len 24576 data: 9 7 6 7 8 5
69932024-02-20T22:44:38.080Z----------------------------------------------------------------
69942024-02-20T22:44:38.080Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69952024-02-20T22:44:38.081ZStates: Active Active Active
69962024-02-20T22:44:38.081ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69972024-02-20T22:44:38.081Z 1 Acked 1000 Write 1 Done Done Done false
69982024-02-20T22:44:38.081Z 2 Acked 1001 Read 7 Done Done Done false
69992024-02-20T22:44:38.081Z 3 Acked 1002 Write 10 Done Done Done false
70002024-02-20T22:44:38.081Z 4 Acked 1003 Write 9 Done Done Done false
70012024-02-20T22:44:38.081Z 5 Acked 1004 Read 9 Done Done Done false
70022024-02-20T22:44:38.081Z 6 Acked 1005 Read 7 Done Done Done false
70032024-02-20T22:44:38.081Z 7 Acked 1006 Read 2 Done Done Done false
70042024-02-20T22:44:38.081Z 8 Acked 1007 Write 3 Done Done Done false
70052024-02-20T22:44:38.081Z 9 Acked 1008 Write 6 Done Done Done false
70062024-02-20T22:44:38.081Z 10 Acked 1009 Write 1 Done Done Done false
70072024-02-20T22:44:38.081Z 11 Acked 1010 Read 8 Done Done Done false
70082024-02-20T22:44:38.081Z 12 Acked 1011 Flush 0 Done Done Sent false
70092024-02-20T22:44:38.081Z 13 Acked 1012 Write 7 Done Done Sent false
70102024-02-20T22:44:38.081Z 14 Acked 1013 Flush 0 Done Done Sent false
70112024-02-20T22:44:38.081Z 15 Acked 1014 Write 3 Done Done Sent false
70122024-02-20T22:44:38.081Z 16 Acked 1015 Read 1 Done Done Sent false
70132024-02-20T22:44:38.081Z 17 Acked 1016 Read 4 Done Done Sent false
70142024-02-20T22:44:38.081Z 18 Acked 1017 Flush 0 Done Done Sent false
70152024-02-20T22:44:38.081Z 19 Acked 1018 Write 6 Done Done Sent false
70162024-02-20T22:44:38.081Z 20 Acked 1019 Write 3 Done Done Sent false
70172024-02-20T22:44:38.081Z 21 Acked 1020 Read 4 Done Done Sent false
70182024-02-20T22:44:38.081Z 22 Acked 1021 Read 10 Done Done Sent false
70192024-02-20T22:44:38.081Z 23 Acked 1022 Write 3 Done Done Sent false
70202024-02-20T22:44:38.082Z 24 Acked 1023 Write 6 Done Done Sent false
70212024-02-20T22:44:38.082Z 25 Acked 1024 Read 3 Done Done Sent false
70222024-02-20T22:44:38.082Z 26 Acked 1025 Read 5 Done Done Sent false
70232024-02-20T22:44:38.082Z 27 Acked 1026 Write 9 Done Done Sent false
70242024-02-20T22:44:38.082Z 28 Acked 1027 Write 8 Done Done Sent false
70252024-02-20T22:44:38.082Z 29 Acked 1028 Write 5 Done Done Sent false
70262024-02-20T22:44:38.082Z 30 Acked 1029 Write 6 Sent Sent Sent false
70272024-02-20T22:44:38.082Z STATES DS:0 DS:1 DS:2 TOTAL
70282024-02-20T22:44:38.082Z New 0 0 0 0
70292024-02-20T22:44:38.082Z Sent 1 1 19 21
70302024-02-20T22:44:38.082Z Done 29 29 11 69
70312024-02-20T22:44:38.082Z Skipped 0 0 0 0
70322024-02-20T22:44:38.082Z Error 0 0 0 0
70332024-02-20T22:44:38.082ZLast Flush: 1017 1017 0
70342024-02-20T22:44:38.082ZDownstairs last five completed:
70352024-02-20T22:44:38.082ZUpstairs last five completed: 30 29 28 27 26
70362024-02-20T22:44:38.082Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:37.608015709Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1394}
70372024-02-20T22:44:38.082ZWrote out file "/tmp/test_repair_verify.data"
70382024-02-20T22:44:38.087Z
70392024-02-20T22:44:38.087ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70402024-02-20T22:44:38.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70412024-02-20T22:44:38.110Z 0 000-019 55 55 55 101 101 101 F F F
70422024-02-20T22:44:38.111Z 1 020-039 58 58 58 105 105 105 T T F <---
70432024-02-20T22:44:38.111Z 2 040-059 58 58 58 105 105 105 T T F <---
70442024-02-20T22:44:38.111Z 3 060-079 64 64 59 112 112 106 F F T <---
70452024-02-20T22:44:38.111Z 4 080-099 64 64 60 112 112 107 F F T <---
70462024-02-20T22:44:38.111Z 5 100-119 60 60 60 107 107 107 T T F <---
70472024-02-20T22:44:38.111Z 6 120-139 64 64 61 113 113 108 T T F <---
70482024-02-20T22:44:38.111Z 7 140-159 64 64 60 113 113 107 T T F <---
70492024-02-20T22:44:38.111Z 8 160-179 60 60 60 107 107 107 F F F
70502024-02-20T22:44:38.111Z 9 180-199 62 62 62 110 110 110 F F F
70512024-02-20T22:44:38.111Z 10 200-219 62 62 62 110 110 110 F F F
70522024-02-20T22:44:38.111Z 11 220-239 63 63 63 111 111 111 F F F
70532024-02-20T22:44:38.115Z 12 240-259 63 63 63 111 111 111 F F F
70542024-02-20T22:44:38.115Z 13 260-279 64 64 50 114 114 96 F F T <---
70552024-02-20T22:44:38.115Z 14 280-299 58 58 58 104 104 104 T T F <---
70562024-02-20T22:44:38.115Z 15 300-319 64 64 61 112 112 108 F F T <---
70572024-02-20T22:44:38.115Z 16 320-339 54 54 54 100 100 100 F F F
70582024-02-20T22:44:38.115Z 17 340-359 51 51 51 97 97 97 T T F <---
70592024-02-20T22:44:38.115Z 18 360-379 62 62 62 110 110 110 F F F
70602024-02-20T22:44:38.115Z 19 380-399 62 62 62 110 110 110 T T F <---
70612024-02-20T22:44:38.115Z 20 400-419 63 63 63 111 111 111 T T F <---
70622024-02-20T22:44:38.115Z 21 420-439 63 63 63 111 111 111 T T F <---
70632024-02-20T22:44:38.115Z 22 440-459 62 62 62 110 110 110 F F F
70642024-02-20T22:44:38.115Z 23 460-479 63 63 63 111 111 111 F F F
70652024-02-20T22:44:38.115Z 24 480-499 62 62 62 110 110 110 F F F
70662024-02-20T22:44:38.115Z 25 500-519 62 62 62 110 110 110 F F F
70672024-02-20T22:44:38.116Z 26 520-539 59 59 59 106 106 106 F F F
70682024-02-20T22:44:38.116Z 27 540-559 64 64 59 112 112 106 F F T <---
70692024-02-20T22:44:38.116Z 28 560-579 64 64 60 112 112 107 T T T <---
70702024-02-20T22:44:38.116Z 29 580-599 60 60 60 107 107 107 T T F <---
70712024-02-20T22:44:38.116ZMax gen: 64, Max flush: 114
70722024-02-20T22:44:38.116ZError: Difference in extent metadata found!
70732024-02-20T22:44:38.116ZA Difference in extent metadata is expected here
70742024-02-20T22:44:38.116ZOn loop 32
70752024-02-20T22:44:38.116Z
70762024-02-20T22:44:38.116ZVerifying data now
70772024-02-20T22:44:42.912ZNo change after verify
70782024-02-20T22:44:42.913ZLoop: 32 Downstairs dump after verify (and repair):
70792024-02-20T22:44:42.917ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70802024-02-20T22:44:42.917Z 0 000-019 55 55 55 101 101 101 F F F
70812024-02-20T22:44:42.917Z 1 020-039 65 65 65 115 115 115 F F F
70822024-02-20T22:44:42.917Z 2 040-059 65 65 65 115 115 115 F F F
70832024-02-20T22:44:42.918Z 3 060-079 64 64 64 112 112 112 F F F
70842024-02-20T22:44:42.918Z 4 080-099 64 64 64 112 112 112 F F F
70852024-02-20T22:44:42.922Z 5 100-119 65 65 65 115 115 115 F F F
70862024-02-20T22:44:42.922Z 6 120-139 65 65 65 115 115 115 F F F
70872024-02-20T22:44:42.922Z 7 140-159 65 65 65 115 115 115 F F F
70882024-02-20T22:44:42.922Z 8 160-179 60 60 60 107 107 107 F F F
70892024-02-20T22:44:42.922Z 9 180-199 62 62 62 110 110 110 F F F
70902024-02-20T22:44:42.922Z 10 200-219 62 62 62 110 110 110 F F F
70912024-02-20T22:44:42.922Z 11 220-239 63 63 63 111 111 111 F F F
70922024-02-20T22:44:42.923Z 12 240-259 63 63 63 111 111 111 F F F
70932024-02-20T22:44:42.923Z 13 260-279 64 64 64 114 114 114 F F F
70942024-02-20T22:44:42.923Z 14 280-299 65 65 65 115 115 115 F F F
70952024-02-20T22:44:42.923Z 15 300-319 64 64 64 112 112 112 F F F
70962024-02-20T22:44:42.923Z 16 320-339 54 54 54 100 100 100 F F F
70972024-02-20T22:44:42.923Z 17 340-359 65 65 65 115 115 115 F F F
70982024-02-20T22:44:42.923Z 18 360-379 62 62 62 110 110 110 F F F
70992024-02-20T22:44:42.923Z 19 380-399 65 65 65 115 115 115 F F F
71002024-02-20T22:44:42.923Z 20 400-419 65 65 65 115 115 115 F F F
71012024-02-20T22:44:42.923Z 21 420-439 65 65 65 115 115 115 F F F
71022024-02-20T22:44:42.923Z 22 440-459 62 62 62 110 110 110 F F F
71032024-02-20T22:44:42.923Z 23 460-479 63 63 63 111 111 111 F F F
71042024-02-20T22:44:42.923Z 24 480-499 62 62 62 110 110 110 F F F
71052024-02-20T22:44:42.923Z 25 500-519 62 62 62 110 110 110 F F F
71062024-02-20T22:44:42.923Z 26 520-539 59 59 59 106 106 106 F F F
71072024-02-20T22:44:42.923Z 27 540-559 64 64 64 112 112 112 F F F
71082024-02-20T22:44:42.923Z 28 560-579 65 65 65 115 115 115 F F F
71092024-02-20T22:44:42.923Z 29 580-599 65 65 65 115 115 115 F F F
71102024-02-20T22:44:42.923ZMax gen: 65, Max flush: 115
71112024-02-20T22:44:42.923Z
71122024-02-20T22:44:42.923ZBegin loop 33
71132024-02-20T22:44:42.923ZDownstairs to restart: 1
71142024-02-20T22:44:42.939Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.467863869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402}
71152024-02-20T22:44:42.946Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.468296203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402}
71162024-02-20T22:44:42.946Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.468335922Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402}
71172024-02-20T22:44:42.946Z{"msg":"Crucible 464efbcc-fa66-4b69-a0eb-84062f0c8bd5 has session id: 2606e774-512c-45bc-a245-82197c60daf6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470215895Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71182024-02-20T22:44:42.946Z{"msg":"Upstairs opts: Upstairs UUID: 464efbcc-fa66-4b69-a0eb-84062f0c8bd5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470254844Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71192024-02-20T22:44:42.946Z{"msg":"Crucible stats registered with UUID: 464efbcc-fa66-4b69-a0eb-84062f0c8bd5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470272723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71202024-02-20T22:44:42.946ZCrucible runtime is spawned
71212024-02-20T22:44:42.946Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470579792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402}
71222024-02-20T22:44:42.946Z{"msg":"464efbcc-fa66-4b69-a0eb-84062f0c8bd5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470668389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71232024-02-20T22:44:42.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470711087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71242024-02-20T22:44:42.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470733746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71252024-02-20T22:44:42.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470761705Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71262024-02-20T22:44:42.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470792684Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71272024-02-20T22:44:42.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470834093Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71282024-02-20T22:44:42.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470860702Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71292024-02-20T22:44:42.946Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470888051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71302024-02-20T22:44:42.946Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470955328Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71312024-02-20T22:44:42.946Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.470990347Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71322024-02-20T22:44:42.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.471112473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71332024-02-20T22:44:42.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.47117374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71342024-02-20T22:44:42.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.471206079Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71352024-02-20T22:44:42.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.471836687Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71362024-02-20T22:44:42.947Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.471872835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71372024-02-20T22:44:42.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.471890225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71382024-02-20T22:44:42.947Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.471907404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71392024-02-20T22:44:42.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.472074118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71402024-02-20T22:44:42.947Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.472130906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71412024-02-20T22:44:42.960Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.472452515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71422024-02-20T22:44:42.960Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.472517803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71432024-02-20T22:44:42.960Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.472886419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71442024-02-20T22:44:42.960Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473109471Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71452024-02-20T22:44:42.960Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.47314693Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71462024-02-20T22:44:42.960Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473163429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71472024-02-20T22:44:42.960Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473177139Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71482024-02-20T22:44:42.961Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473191078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71492024-02-20T22:44:42.961Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473224337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71502024-02-20T22:44:42.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473344673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71512024-02-20T22:44:42.961Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473378032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71522024-02-20T22:44:42.961Z{"msg":"[0]R flush_numbers[0..12]: [101, 115, 115, 112, 112, 115, 115, 115, 107, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473396861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71532024-02-20T22:44:42.961Z{"msg":"[0]R generation[0..12]: [55, 65, 65, 64, 64, 65, 65, 65, 60, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.47341881Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71542024-02-20T22:44:42.961Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.47343379Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71552024-02-20T22:44:42.961Z{"msg":"[1]R flush_numbers[0..12]: [101, 115, 115, 112, 112, 115, 115, 115, 107, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473448159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71562024-02-20T22:44:42.961Z{"msg":"[1]R generation[0..12]: [55, 65, 65, 64, 64, 65, 65, 65, 60, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473469618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71572024-02-20T22:44:42.961Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473484588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71582024-02-20T22:44:42.961Z{"msg":"[2]R flush_numbers[0..12]: [101, 115, 115, 112, 112, 115, 115, 115, 107, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473511717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71592024-02-20T22:44:42.961Z{"msg":"[2]R generation[0..12]: [55, 65, 65, 64, 64, 65, 65, 65, 60, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473526406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71602024-02-20T22:44:42.961Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
71612024-02-20T22:44:42.961Z,"time":"2024-02-20T22:44:42.473552975Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71622024-02-20T22:44:42.961Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473573145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71632024-02-20T22:44:42.961Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473587104Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71642024-02-20T22:44:42.961Z{"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473600964Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71652024-02-20T22:44:42.961Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473627273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71662024-02-20T22:44:42.961Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473641072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71672024-02-20T22:44:42.961Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473659991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71682024-02-20T22:44:42.961Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473673491Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71692024-02-20T22:44:42.961Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473687281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71702024-02-20T22:44:42.961Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.47370143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71712024-02-20T22:44:42.961Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.47371462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71722024-02-20T22:44:42.961Z{"msg":"464efbcc-fa66-4b69-a0eb-84062f0c8bd5 is now active with session: 2606e774-512c-45bc-a245-82197c60daf6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473727879Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71732024-02-20T22:44:42.962Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473748299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402,"session_id":"2606e774-512c-45bc-a245-82197c60daf6"}
71742024-02-20T22:44:42.962Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:42.473761318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402}
71752024-02-20T22:44:42.962ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71762024-02-20T22:44:42.962ZLoading write count information from file "/tmp/test_repair_verify.data"
71772024-02-20T22:44:42.962ZRun Repair workload
71782024-02-20T22:44:42.962Z01/30 Flush
71792024-02-20T22:44:42.962Z02/30 Flush
71802024-02-20T22:44:42.962Z03/30 Write block 166 len 40960 data: 5 5 6 6 7 6 6 6 4 6
71812024-02-20T22:44:42.962Z04/30 Flush
71822024-02-20T22:44:42.967Z05/30 Write block 67 len 12288 data: 5 6 7
71832024-02-20T22:44:42.967Z06/30 Write block 298 len 32768 data: 3 3 2 5 6 6 6 6
71842024-02-20T22:44:42.967Z07/30 Write block 525 len 8192 data: 3 5
71852024-02-20T22:44:42.967Z08/30 Read block 366 len 40960
71862024-02-20T22:44:42.968Z09/30 Read block 48 len 4096
71872024-02-20T22:44:42.968Z10/30 Read block 472 len 40960
71882024-02-20T22:44:42.968Z11/30 Write block 478 len 4096 data: 10
71892024-02-20T22:44:42.968Z12/30 Write block 314 len 12288 data: 6 8 7
71902024-02-20T22:44:42.968Z13/30 Write block 296 len 40960 data: 3 3 4 4 3 6 7 7 7 7
71912024-02-20T22:44:42.968Z14/30 Read block 429 len 20480
71922024-02-20T22:44:42.968Z15/30 Write block 62 len 4096 data: 3
71932024-02-20T22:44:42.968Z16/30 Write block 40 len 24576 data: 6 5 6 5 5 3
71942024-02-20T22:44:42.968Z17/30 Write block 135 len 28672 data: 10 8 7 7 7 8 7
71952024-02-20T22:44:42.968Z18/30 Read block 24 len 24576
71962024-02-20T22:44:42.968Z19/30 Write block 43 len 32768 data: 6 6 4 3 4 5 5 4
71972024-02-20T22:44:42.968Z20/30 Read block 543 len 16384
71982024-02-20T22:44:42.968Z21/30 Read block 310 len 28672
71992024-02-20T22:44:42.968Z22/30 Read block 163 len 8192
72002024-02-20T22:44:42.968Z23/30 Write block 526 len 28672 data: 6 7 6 7 8 6 4
72012024-02-20T22:44:42.973Z24/30 Write block 172 len 8192 data: 7 7
72022024-02-20T22:44:42.973Z25/30 Read block 162 len 24576
72032024-02-20T22:44:42.973Z26/30 Read block 346 len 36864
72042024-02-20T22:44:42.973Z27/30 Read block 203 len 12288
72052024-02-20T22:44:42.973Z28/30 Read block 29 len 32768
72062024-02-20T22:44:42.973Z29/30 Write block 422 len 16384 data: 6 6 4 4
72072024-02-20T22:44:42.973Z30/30 Write block 298 len 24576 data: 5 5 4 7 8 8
72082024-02-20T22:44:42.973Z----------------------------------------------------------------
72092024-02-20T22:44:42.973Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:28
72102024-02-20T22:44:42.973ZStates: Active Active Active
72112024-02-20T22:44:42.973ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72122024-02-20T22:44:42.973Z 3 Acked 1002 Write 10 Done Sent Done false
72132024-02-20T22:44:42.973Z 4 Acked 1003 Flush 0 Done Sent Done false
72142024-02-20T22:44:42.973Z 5 Acked 1004 Write 3 Done Sent Done false
72152024-02-20T22:44:42.973Z 6 Acked 1005 Write 8 Done Sent Done false
72162024-02-20T22:44:42.973Z 7 Acked 1006 Write 2 Done Sent Done false
72172024-02-20T22:44:42.974Z 8 Acked 1007 Read 10 Done Sent Done false
72182024-02-20T22:44:42.974Z 9 Acked 1008 Read 1 Done Sent Done false
72192024-02-20T22:44:42.974Z 10 Acked 1009 Read 10 Done Sent Done false
72202024-02-20T22:44:42.974Z 11 Acked 1010 Write 1 Done Sent Done false
72212024-02-20T22:44:42.974Z 12 Acked 1011 Write 3 Done Sent Done false
72222024-02-20T22:44:42.974Z 13 Acked 1012 Write 10 Done Sent Done false
72232024-02-20T22:44:42.974Z 14 Acked 1013 Read 5 Done Sent Done false
72242024-02-20T22:44:42.974Z 15 Acked 1014 Write 1 Done Sent Done false
72252024-02-20T22:44:42.974Z 16 Acked 1015 Write 6 Done Sent Done false
72262024-02-20T22:44:42.974Z 17 Acked 1016 Write 7 Done Sent Done false
72272024-02-20T22:44:42.974Z 18 Acked 1017 Read 6 Done Sent Done false
72282024-02-20T22:44:42.974Z 19 Acked 1018 Write 8 Done Sent Done false
72292024-02-20T22:44:42.974Z 20 Acked 1019 Read 4 Done Sent Done false
72302024-02-20T22:44:42.974Z 21 Acked 1020 Read 7 Done Sent Done false
72312024-02-20T22:44:42.974Z 22 Acked 1021 Read 2 Done Sent Done false
72322024-02-20T22:44:42.974Z 23 Acked 1022 Write 7 Done Sent Done false
72332024-02-20T22:44:42.974Z 24 Acked 1023 Write 2 Done Sent Done false
72342024-02-20T22:44:42.974Z 25 Acked 1024 Read 6 Done Sent Done false
72352024-02-20T22:44:42.974Z 26 Acked 1025 Read 9 Done Sent Done false
72362024-02-20T22:44:42.974Z 27 Acked 1026 Read 3 Done Sent Done false
72372024-02-20T22:44:42.974Z 28 Acked 1027 Read 8 Done Sent Done false
72382024-02-20T22:44:42.974Z 29 Acked 1028 Write 4 Done Sent Sent false
72392024-02-20T22:44:42.974Z 30 Acked 1029 Write 6 Sent Sent Sent false
72402024-02-20T22:44:42.974Z STATES DS:0 DS:1 DS:2 TOTAL
72412024-02-20T22:44:42.974Z New 0 0 0 0
72422024-02-20T22:44:42.975Z Sent 1 28 2 31
72432024-02-20T22:44:42.975Z Done 27 0 26 53
72442024-02-20T22:44:42.975Z Skipped 0 0 0 0
72452024-02-20T22:44:42.975Z Error 0 0 0 0
72462024-02-20T22:44:42.975ZLast Flush: 1003 1001 1003
72472024-02-20T22:44:42.975ZDownstairs last five completed: 1001 1000
72482024-02-20T22:44:42.975ZUpstairs last five completed: 30 29 28 27 26
72492024-02-20T22:44:42.986Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:42.501910751Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1402}
72502024-02-20T22:44:42.986ZWrote out file "/tmp/test_repair_verify.data"
72512024-02-20T22:44:42.986Z
72522024-02-20T22:44:42.986ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72532024-02-20T22:44:42.997ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72542024-02-20T22:44:42.997Z 0 000-019 55 55 55 101 101 101 F F F
72552024-02-20T22:44:42.997Z 1 020-039 65 65 65 115 115 115 F F F
72562024-02-20T22:44:42.997Z 2 040-059 65 65 65 115 115 115 T F T <---
72572024-02-20T22:44:42.997Z 3 060-079 64 64 64 112 112 112 T F T <---
72582024-02-20T22:44:42.997Z 4 080-099 64 64 64 112 112 112 F F F
72592024-02-20T22:44:42.997Z 5 100-119 65 65 65 115 115 115 F F F
72602024-02-20T22:44:42.997Z 6 120-139 65 65 65 115 115 115 T F T <---
72612024-02-20T22:44:43.003Z 7 140-159 65 65 65 115 115 115 T F T <---
72622024-02-20T22:44:43.003Z 8 160-179 66 60 66 118 107 118 T F T <---
72632024-02-20T22:44:43.003Z 9 180-199 62 62 62 110 110 110 F F F
72642024-02-20T22:44:43.003Z 10 200-219 62 62 62 110 110 110 F F F
72652024-02-20T22:44:43.003Z 11 220-239 63 63 63 111 111 111 F F F
72662024-02-20T22:44:43.003Z 12 240-259 63 63 63 111 111 111 F F F
72672024-02-20T22:44:43.003Z 13 260-279 64 64 64 114 114 114 F F F
72682024-02-20T22:44:43.003Z 14 280-299 65 65 65 115 115 115 T F T <---
72692024-02-20T22:44:43.003Z 15 300-319 64 64 64 112 112 112 T F T <---
72702024-02-20T22:44:43.003Z 16 320-339 54 54 54 100 100 100 F F F
72712024-02-20T22:44:43.003Z 17 340-359 65 65 65 115 115 115 F F F
72722024-02-20T22:44:43.003Z 18 360-379 62 62 62 110 110 110 F F F
72732024-02-20T22:44:43.003Z 19 380-399 65 65 65 115 115 115 F F F
72742024-02-20T22:44:43.003Z 20 400-419 65 65 65 115 115 115 F F F
72752024-02-20T22:44:43.003Z 21 420-439 65 65 65 115 115 115 T F T <---
72762024-02-20T22:44:43.003Z 22 440-459 62 62 62 110 110 110 F F F
72772024-02-20T22:44:43.003Z 23 460-479 63 63 63 111 111 111 T F T <---
72782024-02-20T22:44:43.003Z 24 480-499 62 62 62 110 110 110 F F F
72792024-02-20T22:44:43.003Z 25 500-519 62 62 62 110 110 110 F F F
72802024-02-20T22:44:43.003Z 26 520-539 59 59 59 106 106 106 T F T <---
72812024-02-20T22:44:43.004Z 27 540-559 64 64 64 112 112 112 F F F
72822024-02-20T22:44:43.004Z 28 560-579 65 65 65 115 115 115 F F F
72832024-02-20T22:44:43.004Z 29 580-599 65 65 65 115 115 115 F F F
72842024-02-20T22:44:43.004ZMax gen: 66, Max flush: 118
72852024-02-20T22:44:43.004ZError: Difference in extent metadata found!
72862024-02-20T22:44:43.004ZA Difference in extent metadata is expected here
72872024-02-20T22:44:43.004ZOn loop 33
72882024-02-20T22:44:43.004Z
72892024-02-20T22:44:43.009ZVerifying data now
72902024-02-20T22:44:43.530ZNo change after verify
72912024-02-20T22:44:43.530ZLoop: 33 Downstairs dump after verify (and repair):
72922024-02-20T22:44:43.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72932024-02-20T22:44:43.544Z 0 000-019 55 55 55 101 101 101 F F F
72942024-02-20T22:44:43.544Z 1 020-039 65 65 65 115 115 115 F F F
72952024-02-20T22:44:43.544Z 2 040-059 67 67 67 119 119 119 F F F
72962024-02-20T22:44:43.544Z 3 060-079 67 67 67 119 119 119 F F F
72972024-02-20T22:44:43.544Z 4 080-099 64 64 64 112 112 112 F F F
72982024-02-20T22:44:43.544Z 5 100-119 65 65 65 115 115 115 F F F
72992024-02-20T22:44:43.544Z 6 120-139 67 67 67 119 119 119 F F F
73002024-02-20T22:44:43.544Z 7 140-159 67 67 67 119 119 119 F F F
73012024-02-20T22:44:43.544Z 8 160-179 67 67 67 119 119 119 F F F
73022024-02-20T22:44:43.544Z 9 180-199 62 62 62 110 110 110 F F F
73032024-02-20T22:44:43.544Z 10 200-219 62 62 62 110 110 110 F F F
73042024-02-20T22:44:43.549Z 11 220-239 63 63 63 111 111 111 F F F
73052024-02-20T22:44:43.549Z 12 240-259 63 63 63 111 111 111 F F F
73062024-02-20T22:44:43.549Z 13 260-279 64 64 64 114 114 114 F F F
73072024-02-20T22:44:43.549Z 14 280-299 67 67 67 119 119 119 F F F
73082024-02-20T22:44:43.549Z 15 300-319 67 67 67 119 119 119 F F F
73092024-02-20T22:44:43.549Z 16 320-339 54 54 54 100 100 100 F F F
73102024-02-20T22:44:43.549Z 17 340-359 65 65 65 115 115 115 F F F
73112024-02-20T22:44:43.549Z 18 360-379 62 62 62 110 110 110 F F F
73122024-02-20T22:44:43.549Z 19 380-399 65 65 65 115 115 115 F F F
73132024-02-20T22:44:43.549Z 20 400-419 65 65 65 115 115 115 F F F
73142024-02-20T22:44:43.549Z 21 420-439 67 67 67 119 119 119 F F F
73152024-02-20T22:44:43.549Z 22 440-459 62 62 62 110 110 110 F F F
73162024-02-20T22:44:43.549Z 23 460-479 67 67 67 119 119 119 F F F
73172024-02-20T22:44:43.549Z 24 480-499 62 62 62 110 110 110 F F F
73182024-02-20T22:44:43.549Z 25 500-519 62 62 62 110 110 110 F F F
73192024-02-20T22:44:43.549Z 26 520-539 67 67 67 119 119 119 F F F
73202024-02-20T22:44:43.549Z 27 540-559 64 64 64 112 112 112 F F F
73212024-02-20T22:44:43.549Z 28 560-579 65 65 65 115 115 115 F F F
73222024-02-20T22:44:43.549Z 29 580-599 65 65 65 115 115 115 F F F
73232024-02-20T22:44:43.549ZMax gen: 67, Max flush: 119
73242024-02-20T22:44:43.549Z
73252024-02-20T22:44:43.549ZBegin loop 34
73262024-02-20T22:44:43.549ZDownstairs to restart: 2
73272024-02-20T22:44:43.565Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.093662546Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410}
73282024-02-20T22:44:43.573Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.093914597Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410}
73292024-02-20T22:44:43.573Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.093943016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410}
73302024-02-20T22:44:43.573Z{"msg":"Crucible 3700276e-2fea-42ae-9fd6-af1de098939f has session id: da089625-807e-4005-9c53-db69563642e1","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
73312024-02-20T22:44:43.573Ztime":"2024-02-20T22:44:43.093992194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73322024-02-20T22:44:43.573Z{"msg":"Upstairs opts: Upstairs UUID: 3700276e-2fea-42ae-9fd6-af1de098939f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094051762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73332024-02-20T22:44:43.573Z{"msg":"Crucible stats registered with UUID: 3700276e-2fea-42ae-9fd6-af1de098939f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09408681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73342024-02-20T22:44:43.573Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09410979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410}
73352024-02-20T22:44:43.573Z{"msg":"3700276e-2fea-42ae-9fd6-af1de098939f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094202706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73362024-02-20T22:44:43.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094238975Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73372024-02-20T22:44:43.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094256795Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73382024-02-20T22:44:43.574Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094285343Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73392024-02-20T22:44:43.574Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094309873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73402024-02-20T22:44:43.574Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094340972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73412024-02-20T22:44:43.574Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09436724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73422024-02-20T22:44:43.574Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094411229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73432024-02-20T22:44:43.574Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094443498Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"io task","client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73442024-02-20T22:44:43.574Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094483516Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"io task","client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73452024-02-20T22:44:43.574Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094512615Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73462024-02-20T22:44:43.574Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094546594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"io task","client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73472024-02-20T22:44:43.574Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.094571603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"io task","client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73482024-02-20T22:44:43.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09522835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73492024-02-20T22:44:43.574Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095266548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73502024-02-20T22:44:43.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095283958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73512024-02-20T22:44:43.574Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095316726Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73522024-02-20T22:44:43.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095342296Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73532024-02-20T22:44:43.574Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095368265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73542024-02-20T22:44:43.574Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095876246Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73552024-02-20T22:44:43.574Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095913195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73562024-02-20T22:44:43.574Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.095933744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73572024-02-20T22:44:43.574Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096414637Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73582024-02-20T22:44:43.574Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096451456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73592024-02-20T22:44:43.574Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096468465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73602024-02-20T22:44:43.574Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096530183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73612024-02-20T22:44:43.574Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096600681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73622024-02-20T22:44:43.575Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.0966173Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73632024-02-20T22:44:43.575Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096639209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73642024-02-20T22:44:43.575Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096653739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73652024-02-20T22:44:43.575Z{"msg":"[0]R flush_numbers[0..12]: [101, 115, 119, 119, 112, 115, 119, 119, 119, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096674808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73662024-02-20T22:44:43.575Z{"msg":"[0]R generation[0..12]: [55, 65, 67, 67, 64, 65, 67, 67, 67, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096689468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73672024-02-20T22:44:43.575Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
73682024-02-20T22:44:43.575Z,"time":"2024-02-20T22:44:43.096710687Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73692024-02-20T22:44:43.575Z{"msg":"[1]R flush_numbers[0..12]: [101, 115, 119, 119, 112, 115, 119, 119, 119, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096727276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73702024-02-20T22:44:43.575Z{"msg":"[1]R generation[0..12]: [55, 65, 67, 67, 64, 65, 67, 67, 67, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096753795Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73712024-02-20T22:44:43.575Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096768425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73722024-02-20T22:44:43.575Z{"msg":"[2]R flush_numbers[0..12]: [101, 115, 119, 119, 112, 115, 119, 119, 119, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096782574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73732024-02-20T22:44:43.575Z{"msg":"[2]R generation[0..12]: [55, 65, 67, 67, 64, 65, 67, 67, 67, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096807963Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73742024-02-20T22:44:43.575Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096826032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73752024-02-20T22:44:43.575Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096847202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73762024-02-20T22:44:43.575Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096861001Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73772024-02-20T22:44:43.575Z{"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096873941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73782024-02-20T22:44:43.575Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09688769Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73792024-02-20T22:44:43.575Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09690132Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73802024-02-20T22:44:43.575Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.09691356Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73812024-02-20T22:44:43.575Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096933999Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73822024-02-20T22:44:43.575Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096947988Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"da089625-807e-4005-9c53-db69563642e1"}
73832024-02-20T22:44:43.575Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096961808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73842024-02-20T22:44:43.575Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096981557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73852024-02-20T22:44:43.575Z{"msg":"3700276e-2fea-42ae-9fd6-af1de098939f is now active with session: da089625-807e-4005-9c53-db69563642e1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.096995026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73862024-02-20T22:44:43.575Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.097007196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410,"session_id":"da089625-807e-4005-9c53-db69563642e1"}
73872024-02-20T22:44:43.576Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:43.097019676Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73882024-02-20T22:44:43.576Zip-10-150-1-191.us-west-2.compute.internal","pid":1410}
73892024-02-20T22:44:43.576ZLoading write count information from file "/tmp/test_repair_verify.data"
73902024-02-20T22:44:43.576ZRun Repair workload
73912024-02-20T22:44:43.576Z01/30 Write block 513 len 12288 data: 6 6 7
73922024-02-20T22:44:43.580Z02/30 Write block 176 len 16384 data: 6 6 6 8
73932024-02-20T22:44:43.580Z03/30 Read block 407 len 28672
73942024-02-20T22:44:43.580Z04/30 Read block 91 len 40960
73952024-02-20T22:44:43.580Z05/30 Write block 314 len 36864 data: 7 9 8 9 7 8 9 7 9
73962024-02-20T22:44:43.580Z06/30 Read block 144 len 32768
73972024-02-20T22:44:43.580Z07/30 Flush
73982024-02-20T22:44:43.580Z08/30 Write block 504 len 36864 data: 5 5 5 5 6 6 8 10 6
73992024-02-20T22:44:43.580Z09/30 Read block 543 len 12288
74002024-02-20T22:44:43.587Z10/30 Read block 352 len 12288
74012024-02-20T22:44:43.587Z11/30 Flush
74022024-02-20T22:44:43.587Z12/30 Write block 87 len 20480 data: 4 4 4 4 4
74032024-02-20T22:44:43.594Z13/30 Write block 143 len 40960 data: 8 7 6 4 3 5 7 9 9 8
74042024-02-20T22:44:43.594Z14/30 Write block 289 len 20480 data: 7 5 6 5 4
74052024-02-20T22:44:43.594Z15/30 Read block 517 len 40960
74062024-02-20T22:44:43.594Z16/30 Flush
74072024-02-20T22:44:43.594Z17/30 Write block 148 len 24576 data: 6 8 10 10 9 5
74082024-02-20T22:44:43.594Z18/30 Write block 126 len 20480 data: 3 6 6 6 5
74092024-02-20T22:44:43.594Z19/30 Read block 517 len 16384
74102024-02-20T22:44:43.594Z20/30 Write block 31 len 40960 data: 14 13 13 10 10 8 7 8 9 7
74112024-02-20T22:44:43.601Z21/30 Write block 408 len 4096 data: 7
74122024-02-20T22:44:43.602Z22/30 Read block 562 len 20480
74132024-02-20T22:44:43.602Z23/30 Write block 393 len 12288 data: 9 8 8
74142024-02-20T22:44:43.602Z24/30 Write block 440 len 12288 data: 10 10 10
74152024-02-20T22:44:43.602Z25/30 Read block 536 len 24576
74162024-02-20T22:44:43.602Z26/30 Flush
74172024-02-20T22:44:43.602Z27/30 Write block 581 len 8192 data: 6 7
74182024-02-20T22:44:43.602Z28/30 Read block 210 len 40960
74192024-02-20T22:44:43.606Z29/30 Write block 489 len 8192 data: 3 3
74202024-02-20T22:44:43.606Z30/30 Write block 258 len 36864 data: 3 3 3 3 3 4 7 7 8
74212024-02-20T22:44:43.606Z----------------------------------------------------------------
74222024-02-20T22:44:43.606Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74232024-02-20T22:44:43.606ZStates: Active Active Active
74242024-02-20T22:44:43.606ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74252024-02-20T22:44:43.606Z 1 Acked 1000 Write 3 Done Done Done false
74262024-02-20T22:44:43.606Z 2 Acked 1001 Write 4 Done Done Sent false
74272024-02-20T22:44:43.606Z 3 Acked 1002 Read 7 Done Done Sent false
74282024-02-20T22:44:43.607Z 4 Acked 1003 Read 10 Done Done Sent false
74292024-02-20T22:44:43.607Z 5 Acked 1004 Write 9 Done Done Sent false
74302024-02-20T22:44:43.607Z 6 Acked 1005 Read 8 Done Done Sent false
74312024-02-20T22:44:43.607Z 7 Acked 1006 Flush 0 Done Done Sent false
74322024-02-20T22:44:43.607Z 8 Acked 1007 Write 9 Done Done Sent false
74332024-02-20T22:44:43.607Z 9 Acked 1008 Read 3 Done Done Sent false
74342024-02-20T22:44:43.607Z 10 Acked 1009 Read 3 Done Done Sent false
74352024-02-20T22:44:43.607Z 11 Acked 1010 Flush 0 Done Done Sent false
74362024-02-20T22:44:43.607Z 12 Acked 1011 Write 5 Done Done Sent false
74372024-02-20T22:44:43.607Z 13 Acked 1012 Write 10 Done Done Sent false
74382024-02-20T22:44:43.607Z 14 Acked 1013 Write 5 Done Done Sent false
74392024-02-20T22:44:43.607Z 15 Acked 1014 Read 10 Done Done Sent false
74402024-02-20T22:44:43.607Z 16 Acked 1015 Flush 0 Done Done Sent false
74412024-02-20T22:44:43.607Z 17 Acked 1016 Write 6 Done Done Sent false
74422024-02-20T22:44:43.607Z 18 Acked 1017 Write 5 Done Done Sent false
74432024-02-20T22:44:43.607Z 19 Acked 1018 Read 4 Done Done Sent false
74442024-02-20T22:44:43.607Z 20 Acked 1019 Write 10 Done Done Sent false
74452024-02-20T22:44:43.607Z 21 Acked 1020 Write 1 Done Done Sent false
74462024-02-20T22:44:43.607Z 22 Acked 1021 Read 5 Done Done Sent false
74472024-02-20T22:44:43.607Z 23 Acked 1022 Write 3 Done Done Sent false
74482024-02-20T22:44:43.607Z 24 Acked 1023 Write 3 Done Done Sent false
74492024-02-20T22:44:43.607Z 25 Acked 1024 Read 6 Done Done Sent false
74502024-02-20T22:44:43.607Z 26 Acked 1025 Flush 0 Done Done Sent false
74512024-02-20T22:44:43.607Z 27 Acked 1026 Write 2 Done Done Sent false
74522024-02-20T22:44:43.607Z 28 Acked 1027 Read 10 Done Done Sent false
74532024-02-20T22:44:43.608Z 29 Acked 1028 Write 2 Done Done Sent false
74542024-02-20T22:44:43.608Z 30 Acked 1029 Write 9 Sent Sent Sent false
74552024-02-20T22:44:43.608Z STATES DS:0 DS:1 DS:2 TOTAL
74562024-02-20T22:44:43.608Z New 0 0 0 0
74572024-02-20T22:44:43.608Z Sent 1 1 29 31
74582024-02-20T22:44:43.608Z Done 29 29 1 59
74592024-02-20T22:44:43.608Z Skipped 0 0 0 0
74602024-02-20T22:44:43.608Z Error 0 0 0 0
74612024-02-20T22:44:43.608ZLast Flush: 1025 1025 0
74622024-02-20T22:44:43.608ZDownstairs last five completed:
74632024-02-20T22:44:43.608ZUpstairs last five completed: 30 29 28 27 26
74642024-02-20T22:44:43.608Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:43.130639863Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1410}
74652024-02-20T22:44:43.608ZWrote out file "/tmp/test_repair_verify.data"
74662024-02-20T22:44:43.608Z
74672024-02-20T22:44:43.612ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74682024-02-20T22:44:43.624ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74692024-02-20T22:44:43.624Z 0 000-019 55 55 55 101 101 101 F F F
74702024-02-20T22:44:43.624Z 1 020-039 68 68 65 123 123 115 F F F <---
74712024-02-20T22:44:43.624Z 2 040-059 68 68 67 123 123 119 F F F <---
74722024-02-20T22:44:43.624Z 3 060-079 67 67 67 119 119 119 F F F
74732024-02-20T22:44:43.624Z 4 080-099 68 68 64 122 122 112 F F F <---
74742024-02-20T22:44:43.625Z 5 100-119 65 65 65 115 115 115 F F F
74752024-02-20T22:44:43.625Z 6 120-139 68 68 67 123 123 119 F F F <---
74762024-02-20T22:44:43.625Z 7 140-159 68 68 67 123 123 119 F F F <---
74772024-02-20T22:44:43.625Z 8 160-179 68 68 67 120 120 119 F F F <---
74782024-02-20T22:44:43.625Z 9 180-199 62 62 62 110 110 110 F F F
74792024-02-20T22:44:43.625Z 10 200-219 62 62 62 110 110 110 F F F
74802024-02-20T22:44:43.625Z 11 220-239 63 63 63 111 111 111 F F F
74812024-02-20T22:44:43.625Z 12 240-259 63 63 63 111 111 111 T T F <---
74822024-02-20T22:44:43.625Z 13 260-279 64 64 64 114 114 114 T T F <---
74832024-02-20T22:44:43.629Z 14 280-299 68 68 67 122 122 119 F F F <---
74842024-02-20T22:44:43.629Z 15 300-319 68 68 67 120 120 119 F F F <---
74852024-02-20T22:44:43.630Z 16 320-339 68 68 54 120 120 100 F F F <---
74862024-02-20T22:44:43.630Z 17 340-359 65 65 65 115 115 115 F F F
74872024-02-20T22:44:43.630Z 18 360-379 62 62 62 110 110 110 F F F
74882024-02-20T22:44:43.630Z 19 380-399 68 68 65 123 123 115 F F F <---
74892024-02-20T22:44:43.630Z 20 400-419 68 68 65 123 123 115 F F F <---
74902024-02-20T22:44:43.630Z 21 420-439 67 67 67 119 119 119 F F F
74912024-02-20T22:44:43.630Z 22 440-459 68 68 62 123 123 110 F F F <---
74922024-02-20T22:44:43.630Z 23 460-479 67 67 67 119 119 119 F F F
74932024-02-20T22:44:43.630Z 24 480-499 62 62 62 110 110 110 T T F <---
74942024-02-20T22:44:43.630Z 25 500-519 68 68 62 121 121 110 F F T <---
74952024-02-20T22:44:43.630Z 26 520-539 67 67 67 119 119 119 F F F
74962024-02-20T22:44:43.630Z 27 540-559 64 64 64 112 112 112 F F F
74972024-02-20T22:44:43.630Z 28 560-579 65 65 65 115 115 115 F F F
74982024-02-20T22:44:43.630Z 29 580-599 65 65 65 115 115 115 T T F <---
74992024-02-20T22:44:43.630ZMax gen: 68, Max flush: 123
75002024-02-20T22:44:43.630ZError: Difference in extent metadata found!
75012024-02-20T22:44:43.630ZA Difference in extent metadata is expected here
75022024-02-20T22:44:43.630ZOn loop 34
75032024-02-20T22:44:43.630Z
75042024-02-20T22:44:43.635ZVerifying data now
75052024-02-20T22:44:48.419ZNo change after verify
75062024-02-20T22:44:48.419ZLoop: 34 Downstairs dump after verify (and repair):
75072024-02-20T22:44:48.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75082024-02-20T22:44:48.434Z 0 000-019 55 55 55 101 101 101 F F F
75092024-02-20T22:44:48.434Z 1 020-039 68 68 68 123 123 123 F F F
75102024-02-20T22:44:48.434Z 2 040-059 68 68 68 123 123 123 F F F
75112024-02-20T22:44:48.434Z 3 060-079 67 67 67 119 119 119 F F F
75122024-02-20T22:44:48.435Z 4 080-099 68 68 68 122 122 122 F F F
75132024-02-20T22:44:48.435Z 5 100-119 65 65 65 115 115 115 F F F
75142024-02-20T22:44:48.435Z 6 120-139 68 68 68 123 123 123 F F F
75152024-02-20T22:44:48.435Z 7 140-159 68 68 68 123 123 123 F F F
75162024-02-20T22:44:48.435Z 8 160-179 68 68 68 120 120 120 F F F
75172024-02-20T22:44:48.435Z 9 180-199 62 62 62 110 110 110 F F F
75182024-02-20T22:44:48.435Z 10 200-219 62 62 62 110 110 110 F F F
75192024-02-20T22:44:48.435Z 11 220-239 63 63 63 111 111 111 F F F
75202024-02-20T22:44:48.440Z 12 240-259 69 69 69 124 124 124 F F F
75212024-02-20T22:44:48.440Z 13 260-279 69 69 69 124 124 124 F F F
75222024-02-20T22:44:48.440Z 14 280-299 68 68 68 122 122 122 F F F
75232024-02-20T22:44:48.440Z 15 300-319 68 68 68 120 120 120 F F F
75242024-02-20T22:44:48.440Z 16 320-339 68 68 68 120 120 120 F F F
75252024-02-20T22:44:48.440Z 17 340-359 65 65 65 115 115 115 F F F
75262024-02-20T22:44:48.440Z 18 360-379 62 62 62 110 110 110 F F F
75272024-02-20T22:44:48.440Z 19 380-399 68 68 68 123 123 123 F F F
75282024-02-20T22:44:48.440Z 20 400-419 68 68 68 123 123 123 F F F
75292024-02-20T22:44:48.440Z 21 420-439 67 67 67 119 119 119 F F F
75302024-02-20T22:44:48.440Z 22 440-459 68 68 68 123 123 123 F F F
75312024-02-20T22:44:48.440Z 23 460-479 67 67 67 119 119 119 F F F
75322024-02-20T22:44:48.441Z 24 480-499 69 69 69 124 124 124 F F F
75332024-02-20T22:44:48.441Z 25 500-519 68 68 68 121 121 121 F F F
75342024-02-20T22:44:48.441Z 26 520-539 67 67 67 119 119 119 F F F
75352024-02-20T22:44:48.441Z 27 540-559 64 64 64 112 112 112 F F F
75362024-02-20T22:44:48.441Z 28 560-579 65 65 65 115 115 115 F F F
75372024-02-20T22:44:48.441Z 29 580-599 69 69 69 124 124 124 F F F
75382024-02-20T22:44:48.441ZMax gen: 69, Max flush: 124
75392024-02-20T22:44:48.441Z
75402024-02-20T22:44:48.441ZBegin loop 35
75412024-02-20T22:44:48.441ZDownstairs to restart: 0
75422024-02-20T22:44:48.456Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984306664Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418}
75432024-02-20T22:44:48.465Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984648922Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418}
75442024-02-20T22:44:48.465Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
75452024-02-20T22:44:48.465Z,"time":"2024-02-20T22:44:47.98470447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418}
75462024-02-20T22:44:48.465Z{"msg":"Crucible 5a424056-9dc2-4d20-bcf9-bbd91eb6e591 has session id: 59875c53-5bab-4bf4-8678-4251e7bfc901","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984733709Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75472024-02-20T22:44:48.465Z{"msg":"Upstairs opts: Upstairs UUID: 5a424056-9dc2-4d20-bcf9-bbd91eb6e591, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984762358Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75482024-02-20T22:44:48.465Z{"msg":"Crucible stats registered with UUID: 5a424056-9dc2-4d20-bcf9-bbd91eb6e591","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984790847Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75492024-02-20T22:44:48.465Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984805446Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418}
75502024-02-20T22:44:48.466Z{"msg":"5a424056-9dc2-4d20-bcf9-bbd91eb6e591 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984906793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75512024-02-20T22:44:48.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.984940511Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75522024-02-20T22:44:48.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.98498186Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75532024-02-20T22:44:48.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985007319Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75542024-02-20T22:44:48.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985042908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75552024-02-20T22:44:48.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985070547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75562024-02-20T22:44:48.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985102166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75572024-02-20T22:44:48.466Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985126685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75582024-02-20T22:44:48.466Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985163193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75592024-02-20T22:44:48.466Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985286609Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75602024-02-20T22:44:48.466Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985370786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75612024-02-20T22:44:48.466Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985427374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75622024-02-20T22:44:48.466Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.985457773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75632024-02-20T22:44:48.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.98609894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75642024-02-20T22:44:48.466Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986137198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75652024-02-20T22:44:48.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986153128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75662024-02-20T22:44:48.466Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986175937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75672024-02-20T22:44:48.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986250005Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75682024-02-20T22:44:48.466Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986356281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75692024-02-20T22:44:48.466Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986800925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75702024-02-20T22:44:48.466Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986915411Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75712024-02-20T22:44:48.466Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.986979108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75722024-02-20T22:44:48.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987267838Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75732024-02-20T22:44:48.466Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987307997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75742024-02-20T22:44:48.467Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987334776Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75752024-02-20T22:44:48.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.98749813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75762024-02-20T22:44:48.467Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987536199Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75772024-02-20T22:44:48.467Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987552618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75782024-02-20T22:44:48.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987573837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75792024-02-20T22:44:48.467Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987588977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75802024-02-20T22:44:48.467Z{"msg":"[0]R flush_numbers[0..12]: [101, 123, 123, 119, 122, 115, 123, 123, 120, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987603986Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75812024-02-20T22:44:48.467Z{"msg":"[0]R generation[0..12]: [55, 68, 68, 67, 68, 65, 68, 68, 68, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987625435Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75822024-02-20T22:44:48.467Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987640265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75832024-02-20T22:44:48.467Z{"msg":"[1]R flush_numbers[0..12]: [101, 123, 123, 119, 122, 115, 123, 123, 120, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987662564Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75842024-02-20T22:44:48.467Z{"msg":"[1]R generation[0..12]: [55, 68, 68, 67, 68, 65, 68, 68, 68, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987676663Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75852024-02-20T22:44:48.467ZWait for a query_work_queue command to finish before sending IO
75862024-02-20T22:44:48.467Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987705902Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75872024-02-20T22:44:48.467Z{"msg":"[2]R flush_numbers[0..12]: [101, 123, 123, 119, 122, 115, 123, 123, 120, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987721192Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75882024-02-20T22:44:48.467Z{"msg":"[2]R generation[0..12]: [55, 68, 68, 67, 68, 65, 68, 68, 68, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987734851Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75892024-02-20T22:44:48.467Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987748941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75902024-02-20T22:44:48.467Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.98777508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75912024-02-20T22:44:48.467Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987797979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75922024-02-20T22:44:48.467Z{"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987834608Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75932024-02-20T22:44:48.467Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987864707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75942024-02-20T22:44:48.467Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987880446Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75952024-02-20T22:44:48.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987907905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75962024-02-20T22:44:48.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987945234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75972024-02-20T22:44:48.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987974203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75982024-02-20T22:44:48.468Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.987989622Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
75992024-02-20T22:44:48.468Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.988018871Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
76002024-02-20T22:44:48.468Z{"msg":"5a424056-9dc2-4d20-bcf9-bbd91eb6e591 is now active with session: 59875c53-5bab-4bf4-8678-4251e7bfc901","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.98805581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
76012024-02-20T22:44:48.468Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.988082859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418,"session_id":"59875c53-5bab-4bf4-8678-4251e7bfc901"}
76022024-02-20T22:44:48.468Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:47.988115258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418}
76032024-02-20T22:44:48.468ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76042024-02-20T22:44:48.468ZLoading write count information from file "/tmp/test_repair_verify.data"
76052024-02-20T22:44:48.468ZRun Repair workload
76062024-02-20T22:44:48.473Z01/30 Write block 589 len 40960 data: 7 9 9 8 8 9 9 8 8 6
76072024-02-20T22:44:48.473Z02/30 Read block 364 len 36864
76082024-02-20T22:44:48.473Z03/30 Write block 409 len 28672 data: 10 10 10 10 9 9 8
76092024-02-20T22:44:48.473Z04/30 Read block 138 len 32768
76102024-02-20T22:44:48.473Z05/30 Read block 14 len 8192
76112024-02-20T22:44:48.473Z06/30 Write block 461 len 36864 data: 9 11 11 9 10 9 9 10 11
76122024-02-20T22:44:48.473Z07/30 Read block 306 len 32768
76132024-02-20T22:44:48.473Z08/30 Read block 201 len 40960
76142024-02-20T22:44:48.473Z09/30 Read block 531 len 16384
76152024-02-20T22:44:48.477Z10/30 Write block 491 len 28672 data: 3 5 7 8 8 9 12
76162024-02-20T22:44:48.477Z11/30 Flush
76172024-02-20T22:44:48.477Z12/30 Write block 301 len 36864 data: 8 9 9 8 8 8 8 7 8
76182024-02-20T22:44:48.481Z13/30 Read block 214 len 16384
76192024-02-20T22:44:48.481Z14/30 Write block 10 len 28672 data: 6 5 3 4 3 4 6
76202024-02-20T22:44:48.481Z15/30 Write block 417 len 24576 data: 4 5 7 7 6 7
76212024-02-20T22:44:48.481Z16/30 Write block 346 len 8192 data: 5 7
76222024-02-20T22:44:48.481Z17/30 Read block 14 len 24576
76232024-02-20T22:44:48.486Z18/30 Read block 479 len 20480
76242024-02-20T22:44:48.486Z19/30 Write block 396 len 16384 data: 9 11 7 7
76252024-02-20T22:44:48.487Z20/30 Read block 401 len 4096
76262024-02-20T22:44:48.487Z21/30 Write block 428 len 8192 data: 5 5
76272024-02-20T22:44:48.487Z22/30 Write block 477 len 8192 data: 11 11
76282024-02-20T22:44:48.487Z23/30 Read block 165 len 36864
76292024-02-20T22:44:48.487Z24/30 Write block 24 len 20480 data: 4 9 8 12 13
76302024-02-20T22:44:48.493Z25/30 Write block 505 len 20480 data: 6 6 6 7 7
76312024-02-20T22:44:48.494Z26/30 Write block 487 len 8192 data: 5 5
76322024-02-20T22:44:48.494Z27/30 Read block 486 len 28672
76332024-02-20T22:44:48.494Z28/30 Write block 325 len 40960 data: 5 6 6 6 6 6 6 8 9 8
76342024-02-20T22:44:48.494Z29/30 Write block 264 len 28672 data: 8 8 9 6 6 6 5
76352024-02-20T22:44:48.499Z30/30 Write block 491 len 20480 data: 4 6 8 9 9
76362024-02-20T22:44:48.499Z----------------------------------------------------------------
76372024-02-20T22:44:48.499Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76382024-02-20T22:44:48.499ZStates: Active Active Active
76392024-02-20T22:44:48.499ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76402024-02-20T22:44:48.499Z 1 Acked 1000 Write 10 Sent Done Done false
76412024-02-20T22:44:48.499Z 2 Acked 1001 Read 9 Sent Done Done false
76422024-02-20T22:44:48.499Z 3 Acked 1002 Write 7 Sent Done Done false
76432024-02-20T22:44:48.499Z 4 Acked 1003 Read 8 Sent Done Done false
76442024-02-20T22:44:48.499Z 5 Acked 1004 Read 2 Sent Done Done false
76452024-02-20T22:44:48.499Z 6 Acked 1005 Write 9 Sent Done Done false
76462024-02-20T22:44:48.500Z 7 Acked 1006 Read 8 Sent Done Done false
76472024-02-20T22:44:48.500Z 8 Acked 1007 Read 10 Sent Done Done false
76482024-02-20T22:44:48.500Z 9 Acked 1008 Read 4 Sent Done Done false
76492024-02-20T22:44:48.500Z 10 Acked 1009 Write 7 Sent Done Done false
76502024-02-20T22:44:48.500Z 11 Acked 1010 Flush 0 Sent Done Done false
76512024-02-20T22:44:48.500Z 12 Acked 1011 Write 9 Sent Done Done false
76522024-02-20T22:44:48.500Z 13 Acked 1012 Read 4 Sent Done Done false
76532024-02-20T22:44:48.500Z 14 Acked 1013 Write 7 Sent Done Done false
76542024-02-20T22:44:48.500Z 15 Acked 1014 Write 6 Sent Done Done false
76552024-02-20T22:44:48.500Z 16 Acked 1015 Write 2 Sent Done Done false
76562024-02-20T22:44:48.500Z 17 Acked 1016 Read 6 Sent Done Done false
76572024-02-20T22:44:48.500Z 18 Acked 1017 Read 5 Sent Done Done false
76582024-02-20T22:44:48.500Z 19 Acked 1018 Write 4 Sent Done Done false
76592024-02-20T22:44:48.500Z 20 Acked 1019 Read 1 Sent Done Done false
76602024-02-20T22:44:48.500Z 21 Acked 1020 Write 2 Sent Done Done false
76612024-02-20T22:44:48.500Z 22 Acked 1021 Write 2 Sent Done Done false
76622024-02-20T22:44:48.500Z 23 Acked 1022 Read 9 Sent Done Done false
76632024-02-20T22:44:48.500Z 24 Acked 1023 Write 5 Sent Done Done false
76642024-02-20T22:44:48.500Z 25 Acked 1024 Write 5 Sent Done Done false
76652024-02-20T22:44:48.500Z 26 Acked 1025 Write 2 Sent Done Done false
76662024-02-20T22:44:48.500Z 27 Acked 1026 Read 7 Sent Done Done false
76672024-02-20T22:44:48.500Z 28 Acked 1027 Write 10 Sent Done Done false
76682024-02-20T22:44:48.500Z 29 Acked 1028 Write 7 Sent Sent Sent false
76692024-02-20T22:44:48.500Z 30 Acked 1029 Write 5 Sent Sent Sent false
76702024-02-20T22:44:48.500Z STATES DS:0 DS:1 DS:2 TOTAL
76712024-02-20T22:44:48.501Z New 0 0 0 0
76722024-02-20T22:44:48.501Z Sent 30 2 2 34
76732024-02-20T22:44:48.501Z Done 0 28 28 56
76742024-02-20T22:44:48.501Z Skipped 0 0 0 0
76752024-02-20T22:44:48.501Z Error 0 0 0 0
76762024-02-20T22:44:48.501ZLast Flush: 0 1010 1010
76772024-02-20T22:44:48.501ZDownstairs last five completed:
76782024-02-20T22:44:48.501ZUpstairs last five completed: 30 29 28 27 26
76792024-02-20T22:44:48.501Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:48.020121682Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1418}
76802024-02-20T22:44:48.501ZWrote out file "/tmp/test_repair_verify.data"
76812024-02-20T22:44:48.501Z
76822024-02-20T22:44:48.506ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76832024-02-20T22:44:48.518ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76842024-02-20T22:44:48.518Z 0 000-019 55 55 55 101 101 101 F T T <---
76852024-02-20T22:44:48.518Z 1 020-039 68 68 68 123 123 123 F T T <---
76862024-02-20T22:44:48.518Z 2 040-059 68 68 68 123 123 123 F F F
76872024-02-20T22:44:48.518Z 3 060-079 67 67 67 119 119 119 F F F
76882024-02-20T22:44:48.518Z 4 080-099 68 68 68 122 122 122 F F F
76892024-02-20T22:44:48.518Z 5 100-119 65 65 65 115 115 115 F F F
76902024-02-20T22:44:48.518Z 6 120-139 68 68 68 123 123 123 F F F
76912024-02-20T22:44:48.518Z 7 140-159 68 68 68 123 123 123 F F F
76922024-02-20T22:44:48.518Z 8 160-179 68 68 68 120 120 120 F F F
76932024-02-20T22:44:48.523Z 9 180-199 62 62 62 110 110 110 F F F
76942024-02-20T22:44:48.523Z 10 200-219 62 62 62 110 110 110 F F F
76952024-02-20T22:44:48.523Z 11 220-239 63 63 63 111 111 111 F F F
76962024-02-20T22:44:48.523Z 12 240-259 69 69 69 124 124 124 F F F
76972024-02-20T22:44:48.523Z 13 260-279 69 69 69 124 124 124 F T T <---
76982024-02-20T22:44:48.524Z 14 280-299 68 68 68 122 122 122 F F F
76992024-02-20T22:44:48.524Z 15 300-319 68 68 68 120 120 120 F T T <---
77002024-02-20T22:44:48.524Z 16 320-339 68 68 68 120 120 120 F T T <---
77012024-02-20T22:44:48.524Z 17 340-359 65 65 65 115 115 115 F T T <---
77022024-02-20T22:44:48.524Z 18 360-379 62 62 62 110 110 110 F F F
77032024-02-20T22:44:48.524Z 19 380-399 68 68 68 123 123 123 F T T <---
77042024-02-20T22:44:48.524Z 20 400-419 68 70 70 123 125 125 F T T <---
77052024-02-20T22:44:48.524Z 21 420-439 67 67 67 119 119 119 F T T <---
77062024-02-20T22:44:48.524Z 22 440-459 68 68 68 123 123 123 F F F
77072024-02-20T22:44:48.524Z 23 460-479 67 70 70 119 125 125 F T T <---
77082024-02-20T22:44:48.524Z 24 480-499 69 70 70 124 125 125 F T T <---
77092024-02-20T22:44:48.524Z 25 500-519 68 68 68 121 121 121 F T T <---
77102024-02-20T22:44:48.524Z 26 520-539 67 67 67 119 119 119 F F F
77112024-02-20T22:44:48.524Z 27 540-559 64 64 64 112 112 112 F F F
77122024-02-20T22:44:48.524Z 28 560-579 65 65 65 115 115 115 F F F
77132024-02-20T22:44:48.524Z 29 580-599 69 70 70 124 125 125 F F F <---
77142024-02-20T22:44:48.524ZMax gen: 70, Max flush: 125
77152024-02-20T22:44:48.524ZError: Difference in extent metadata found!
77162024-02-20T22:44:48.524ZA Difference in extent metadata is expected here
77172024-02-20T22:44:48.524ZOn loop 35
77182024-02-20T22:44:48.524Z
77192024-02-20T22:44:48.529ZVerifying data now
77202024-02-20T22:44:53.160ZNo change after verify
77212024-02-20T22:44:53.160ZLoop: 35 Downstairs dump after verify (and repair):
77222024-02-20T22:44:53.174ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77232024-02-20T22:44:53.174Z 0 000-019 71 71 71 126 126 126 F F F
77242024-02-20T22:44:53.174Z 1 020-039 71 71 71 126 126 126 F F F
77252024-02-20T22:44:53.174Z 2 040-059 68 68 68 123 123 123 F F F
77262024-02-20T22:44:53.174Z 3 060-079 67 67 67 119 119 119 F F F
77272024-02-20T22:44:53.174Z 4 080-099 68 68 68 122 122 122 F F F
77282024-02-20T22:44:53.174Z 5 100-119 65 65 65 115 115 115 F F F
77292024-02-20T22:44:53.175Z 6 120-139 68 68 68 123 123 123 F F F
77302024-02-20T22:44:53.175Z 7 140-159 68 68 68 123 123 123 F F F
77312024-02-20T22:44:53.179Z 8 160-179 68 68 68 120 120 120 F F F
77322024-02-20T22:44:53.179Z 9 180-199 62 62 62 110 110 110 F F F
77332024-02-20T22:44:53.179Z 10 200-219 62 62 62 110 110 110 F F F
77342024-02-20T22:44:53.179Z 11 220-239 63 63 63 111 111 111 F F F
77352024-02-20T22:44:53.180Z 12 240-259 69 69 69 124 124 124 F F F
77362024-02-20T22:44:53.180Z 13 260-279 71 71 71 126 126 126 F F F
77372024-02-20T22:44:53.180Z 14 280-299 68 68 68 122 122 122 F F F
77382024-02-20T22:44:53.180Z 15 300-319 71 71 71 126 126 126 F F F
77392024-02-20T22:44:53.180Z 16 320-339 71 71 71 126 126 126 F F F
77402024-02-20T22:44:53.180Z 17 340-359 71 71 71 126 126 126 F F F
77412024-02-20T22:44:53.180Z 18 360-379 62 62 62 110 110 110 F F F
77422024-02-20T22:44:53.180Z 19 380-399 71 71 71 126 126 126 F F F
77432024-02-20T22:44:53.180Z 20 400-419 71 71 71 126 126 126 F F F
77442024-02-20T22:44:53.180Z 21 420-439 71 71 71 126 126 126 F F F
77452024-02-20T22:44:53.180Z 22 440-459 68 68 68 123 123 123 F F F
77462024-02-20T22:44:53.180Z 23 460-479 71 71 71 126 126 126 F F F
77472024-02-20T22:44:53.180Z 24 480-499 71 71 71 126 126 126 F F F
77482024-02-20T22:44:53.180Z 25 500-519 71 71 71 126 126 126 F F F
77492024-02-20T22:44:53.180Z 26 520-539 67 67 67 119 119 119 F F F
77502024-02-20T22:44:53.180Z 27 540-559 64 64 64 112 112 112 F F F
77512024-02-20T22:44:53.180Z 28 560-579 65 65 65 115 115 115 F F F
77522024-02-20T22:44:53.180Z 29 580-599 70 70 70 125 125 125 F F F
77532024-02-20T22:44:53.180ZMax gen: 71, Max flush: 126
77542024-02-20T22:44:53.180Z
77552024-02-20T22:44:53.180ZBegin loop 36
77562024-02-20T22:44:53.180ZDownstairs to restart: 0
77572024-02-20T22:44:53.196Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724228395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426}
77582024-02-20T22:44:53.197Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724493885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426}
77592024-02-20T22:44:53.197Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724531444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426}
77602024-02-20T22:44:53.205Z{"msg":"Crucible 4fb3ca15-dfa3-4f6e-8895-0c1ce4606a21 has session id: bc4f432a-0dec-4664-839b-f985c0e2719d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724558063Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77612024-02-20T22:44:53.205Z{"msg":"Upstairs opts: Upstairs UUID: 4fb3ca15-dfa3-4f6e-8895-0c1ce4606a21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724607441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77622024-02-20T22:44:53.205Z{"msg":"Crucible stats registered with UUID: 4fb3ca15-dfa3-4f6e-8895-0c1ce4606a21","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.72463704Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77632024-02-20T22:44:53.205ZCrucible runtime is spawned
77642024-02-20T22:44:53.205Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724862412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426}
77652024-02-20T22:44:53.205Z{"msg":"4fb3ca15-dfa3-4f6e-8895-0c1ce4606a21 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724974048Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77662024-02-20T22:44:53.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.724994108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77672024-02-20T22:44:53.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725009507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77682024-02-20T22:44:53.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725027386Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77692024-02-20T22:44:53.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725064335Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77702024-02-20T22:44:53.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725104464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77712024-02-20T22:44:53.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725127373Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77722024-02-20T22:44:53.206Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725146332Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"io task","client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77732024-02-20T22:44:53.206Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725189391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"io task","client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77742024-02-20T22:44:53.206Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.72521793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"io task","client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77752024-02-20T22:44:53.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725320626Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"io task","client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77762024-02-20T22:44:53.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725382944Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"io task","client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77772024-02-20T22:44:53.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.725415032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"io task","client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77782024-02-20T22:44:53.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726005022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77792024-02-20T22:44:53.206Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.72605214Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77802024-02-20T22:44:53.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726076519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77812024-02-20T22:44:53.206Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726100448Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77822024-02-20T22:44:53.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726167416Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77832024-02-20T22:44:53.206Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726242953Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77842024-02-20T22:44:53.206Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726655318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77852024-02-20T22:44:53.206Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726694687Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77862024-02-20T22:44:53.206Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.726765174Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77872024-02-20T22:44:53.206Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727209178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77882024-02-20T22:44:53.206Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727270456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77892024-02-20T22:44:53.206Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727311915Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77902024-02-20T22:44:53.206Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727333774Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77912024-02-20T22:44:53.206Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727366073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77922024-02-20T22:44:53.207Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727386702Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77932024-02-20T22:44:53.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727414791Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77942024-02-20T22:44:53.207Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.72743277Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77952024-02-20T22:44:53.207Z{"msg":"[0]R flush_numbers[0..12]: [126, 126, 123, 119, 122, 115, 123, 123, 120, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727463149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77962024-02-20T22:44:53.207Z{"msg":"[0]R generation[0..12]: [71, 71, 68, 67, 68, 65, 68, 68, 68, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727485038Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77972024-02-20T22:44:53.207Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
77982024-02-20T22:44:53.207Z,"time":"2024-02-20T22:44:52.727514908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
77992024-02-20T22:44:53.207Z{"msg":"[1]R flush_numbers[0..12]: [126, 126, 123, 119, 122, 115, 123, 123, 120, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727536187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78002024-02-20T22:44:53.207Z{"msg":"[1]R generation[0..12]: [71, 71, 68, 67, 68, 65, 68, 68, 68, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727565276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78012024-02-20T22:44:53.207Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727584155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78022024-02-20T22:44:53.207Z{"msg":"[2]R flush_numbers[0..12]: [126, 126, 123, 119, 122, 115, 123, 123, 120, 110, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727614124Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78032024-02-20T22:44:53.207Z{"msg":"[2]R generation[0..12]: [71, 71, 68, 67, 68, 65, 68, 68, 68, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727633183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78042024-02-20T22:44:53.207Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727662802Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78052024-02-20T22:44:53.207Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727681031Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78062024-02-20T22:44:53.207Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727709621Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78072024-02-20T22:44:53.207Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.72772685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78082024-02-20T22:44:53.207Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727755379Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78092024-02-20T22:44:53.207Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727772728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78102024-02-20T22:44:53.207Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727794678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78112024-02-20T22:44:53.207Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727830766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78122024-02-20T22:44:53.207Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727855755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78132024-02-20T22:44:53.207Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727876715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78142024-02-20T22:44:53.207Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"nameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78152024-02-20T22:44:53.207Z":"crucible","level":30,"time":"2024-02-20T22:44:52.727905134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78162024-02-20T22:44:53.207Z{"msg":"4fb3ca15-dfa3-4f6e-8895-0c1ce4606a21 is now active with session: bc4f432a-0dec-4664-839b-f985c0e2719d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727920073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78172024-02-20T22:44:53.208Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727945252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426,"session_id":"bc4f432a-0dec-4664-839b-f985c0e2719d"}
78182024-02-20T22:44:53.208Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:52.727976631Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426}
78192024-02-20T22:44:53.208ZLoading write count information from file "/tmp/test_repair_verify.data"
78202024-02-20T22:44:53.208ZRun Repair workload
78212024-02-20T22:44:53.208Z01/30 Write block 457 len 32768 data: 12 12 10 9 10 12 12 10
78222024-02-20T22:44:53.212Z02/30 Read block 376 len 16384
78232024-02-20T22:44:53.212Z03/30 Read block 391 len 32768
78242024-02-20T22:44:53.212Z04/30 Write block 46 len 40960 data: 4 5 6 6 5 5 7 5 6 6
78252024-02-20T22:44:53.212Z05/30 Write block 561 len 8192 data: 4 3
78262024-02-20T22:44:53.212Z06/30 Read block 436 len 40960
78272024-02-20T22:44:53.213Z07/30 Read block 530 len 8192
78282024-02-20T22:44:53.213Z08/30 Write block 229 len 40960 data: 8 10 9 10 12 12 8 8 8 5
78292024-02-20T22:44:53.213Z09/30 Write block 536 len 24576 data: 9 7 8 6 7 8
78302024-02-20T22:44:53.213Z10/30 Write block 89 len 36864 data: 5 5 5 2 2 2 2 2 2
78312024-02-20T22:44:53.213Z11/30 Flush
78322024-02-20T22:44:53.217Z12/30 Write block 577 len 32768 data: 6 6 8 8 7 8 7 5
78332024-02-20T22:44:53.217Z13/30 Write block 365 len 36864 data: 14 12 8 7 6 9 10 9 7
78342024-02-20T22:44:53.217Z14/30 Write block 190 len 16384 data: 5 5 4 4
78352024-02-20T22:44:53.221Z15/30 Read block 366 len 24576
78362024-02-20T22:44:53.221Z16/30 Write block 215 len 8192 data: 2 2
78372024-02-20T22:44:53.221Z17/30 Write block 207 len 8192 data: 5 5
78382024-02-20T22:44:53.221Z18/30 Flush
78392024-02-20T22:44:53.225Z19/30 Write block 367 len 28672 data: 9 8 7 10 11 10 8
78402024-02-20T22:44:53.225Z20/30 Read block 113 len 24576
78412024-02-20T22:44:53.225Z21/30 Flush
78422024-02-20T22:44:53.229Z22/30 Write block 304 len 8192 data: 9 9
78432024-02-20T22:44:53.229Z23/30 Read block 3 len 12288
78442024-02-20T22:44:53.229Z24/30 Read block 283 len 32768
78452024-02-20T22:44:53.229Z25/30 Write block 266 len 16384 data: 10 7 7 7
78462024-02-20T22:44:53.229Z26/30 Read block 94 len 28672
78472024-02-20T22:44:53.234Z27/30 Write block 492 len 8192 data: 7 9
78482024-02-20T22:44:53.234Z28/30 Read block 457 len 24576
78492024-02-20T22:44:53.234Z29/30 Read block 300 len 12288
78502024-02-20T22:44:53.234Z30/30 Read block 136 len 16384
78512024-02-20T22:44:53.234Z----------------------------------------------------------------
78522024-02-20T22:44:53.234Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78532024-02-20T22:44:53.234ZStates: Active Active Active
78542024-02-20T22:44:53.235ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78552024-02-20T22:44:53.235Z 1 Acked 1000 Write 8 Done Done Done false
78562024-02-20T22:44:53.235Z 2 Acked 1001 Read 4 Done Done Done false
78572024-02-20T22:44:53.235Z 3 Acked 1002 Read 8 Sent Done Done false
78582024-02-20T22:44:53.235Z 4 Acked 1003 Write 10 Sent Done Done false
78592024-02-20T22:44:53.235Z 5 Acked 1004 Write 2 Sent Done Done false
78602024-02-20T22:44:53.235Z 6 Acked 1005 Read 10 Sent Done Done false
78612024-02-20T22:44:53.235Z 7 Acked 1006 Read 2 Sent Done Done false
78622024-02-20T22:44:53.235Z 8 Acked 1007 Write 10 Sent Done Done false
78632024-02-20T22:44:53.235Z 9 Acked 1008 Write 6 Sent Done Done false
78642024-02-20T22:44:53.235Z 10 Acked 1009 Write 9 Sent Done Done false
78652024-02-20T22:44:53.235Z 11 Acked 1010 Flush 0 Sent Done Done false
78662024-02-20T22:44:53.235Z 12 Acked 1011 Write 8 Sent Done Done false
78672024-02-20T22:44:53.235Z 13 Acked 1012 Write 9 Sent Done Done false
78682024-02-20T22:44:53.235Z 14 Acked 1013 Write 4 Sent Done Done false
78692024-02-20T22:44:53.235Z 15 Acked 1014 Read 6 Sent Done Done false
78702024-02-20T22:44:53.235Z 16 Acked 1015 Write 2 Sent Done Done false
78712024-02-20T22:44:53.235Z 17 Acked 1016 Write 2 Sent Done Done false
78722024-02-20T22:44:53.235Z 18 Acked 1017 Flush 0 Sent Done Done false
78732024-02-20T22:44:53.235Z 19 Acked 1018 Write 7 Sent Done Done false
78742024-02-20T22:44:53.235Z 20 Acked 1019 Read 6 Sent Done Done false
78752024-02-20T22:44:53.235Z 21 Acked 1020 Flush 0 Sent Done Done false
78762024-02-20T22:44:53.235Z 22 Acked 1021 Write 2 Sent Done Done false
78772024-02-20T22:44:53.235Z 23 Acked 1022 Read 3 Sent Done Done false
78782024-02-20T22:44:53.235Z 24 Acked 1023 Read 8 Sent Done Done false
78792024-02-20T22:44:53.236Z 25 Acked 1024 Write 4 Sent Done Done false
78802024-02-20T22:44:53.236Z 26 Acked 1025 Read 7 Sent Done Done false
78812024-02-20T22:44:53.236Z 27 Acked 1026 Write 2 Sent Done Done false
78822024-02-20T22:44:53.236Z 28 Acked 1027 Read 6 Sent Done Done false
78832024-02-20T22:44:53.236Z 29 Acked 1028 Read 3 Sent Done Done false
78842024-02-20T22:44:53.236Z 30 Acked 1029 Read 4 Sent Done Done false
78852024-02-20T22:44:53.236Z STATES DS:0 DS:1 DS:2 TOTAL
78862024-02-20T22:44:53.236Z New 0 0 0 0
78872024-02-20T22:44:53.236Z Sent 28 0 0 28
78882024-02-20T22:44:53.236Z Done 2 30 30 62
78892024-02-20T22:44:53.236Z Skipped 0 0 0 0
78902024-02-20T22:44:53.236Z Error 0 0 0 0
78912024-02-20T22:44:53.236ZLast Flush: 0 1020 1020
78922024-02-20T22:44:53.236ZDownstairs last five completed:
78932024-02-20T22:44:53.236ZUpstairs last five completed: 30 29 28 27 26
78942024-02-20T22:44:53.236Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:52.7607156Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1426}
78952024-02-20T22:44:53.236ZWrote out file "/tmp/test_repair_verify.data"
78962024-02-20T22:44:53.241Z
78972024-02-20T22:44:53.241ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78982024-02-20T22:44:53.255ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78992024-02-20T22:44:53.255Z 0 000-019 71 71 71 126 126 126 F F F
79002024-02-20T22:44:53.255Z 1 020-039 71 71 71 126 126 126 F F F
79012024-02-20T22:44:53.255Z 2 040-059 68 72 72 123 127 127 F F F <---
79022024-02-20T22:44:53.255Z 3 060-079 67 67 67 119 119 119 F F F
79032024-02-20T22:44:53.255Z 4 080-099 68 72 72 122 127 127 F F F <---
79042024-02-20T22:44:53.255Z 5 100-119 65 65 65 115 115 115 F F F
79052024-02-20T22:44:53.256Z 6 120-139 68 68 68 123 123 123 F F F
79062024-02-20T22:44:53.256Z 7 140-159 68 68 68 123 123 123 F F F
79072024-02-20T22:44:53.256Z 8 160-179 68 68 68 120 120 120 F F F
79082024-02-20T22:44:53.256Z 9 180-199 62 72 72 110 128 128 F F F <---
79092024-02-20T22:44:53.260Z 10 200-219 62 72 72 110 128 128 F F F <---
79102024-02-20T22:44:53.260Z 11 220-239 63 72 72 111 127 127 F F F <---
79112024-02-20T22:44:53.260Z 12 240-259 69 69 69 124 124 124 F F F
79122024-02-20T22:44:53.260Z 13 260-279 71 71 71 126 126 126 F T T <---
79132024-02-20T22:44:53.260Z 14 280-299 68 68 68 122 122 122 F F F
79142024-02-20T22:44:53.260Z 15 300-319 71 71 71 126 126 126 F T T <---
79152024-02-20T22:44:53.261Z 16 320-339 71 71 71 126 126 126 F F F
79162024-02-20T22:44:53.261Z 17 340-359 71 71 71 126 126 126 F F F
79172024-02-20T22:44:53.261Z 18 360-379 62 72 72 110 129 129 F F F <---
79182024-02-20T22:44:53.261Z 19 380-399 71 71 71 126 126 126 F F F
79192024-02-20T22:44:53.261Z 20 400-419 71 71 71 126 126 126 F F F
79202024-02-20T22:44:53.261Z 21 420-439 71 71 71 126 126 126 F F F
79212024-02-20T22:44:53.261Z 22 440-459 68 72 72 123 127 127 T F F <---
79222024-02-20T22:44:53.261Z 23 460-479 71 72 72 126 127 127 T F F <---
79232024-02-20T22:44:53.261Z 24 480-499 71 71 71 126 126 126 F T T <---
79242024-02-20T22:44:53.261Z 25 500-519 71 71 71 126 126 126 F F F
79252024-02-20T22:44:53.261Z 26 520-539 67 72 72 119 127 127 F F F <---
79262024-02-20T22:44:53.261Z 27 540-559 64 72 72 112 127 127 F F F <---
79272024-02-20T22:44:53.261Z 28 560-579 65 72 72 115 128 128 F F F <---
79282024-02-20T22:44:53.261Z 29 580-599 70 72 72 125 128 128 F F F <---
79292024-02-20T22:44:53.261ZMax gen: 72, Max flush: 129
79302024-02-20T22:44:53.261ZError: Difference in extent metadata found!
79312024-02-20T22:44:53.261ZA Difference in extent metadata is expected here
79322024-02-20T22:44:53.261ZOn loop 36
79332024-02-20T22:44:53.261Z
79342024-02-20T22:44:53.266ZVerifying data now
79352024-02-20T22:44:57.963ZNo change after verify
79362024-02-20T22:44:57.963ZLoop: 36 Downstairs dump after verify (and repair):
79372024-02-20T22:44:57.977ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79382024-02-20T22:44:57.977Z 0 000-019 71 71 71 126 126 126 F F F
79392024-02-20T22:44:57.977Z 1 020-039 71 71 71 126 126 126 F F F
79402024-02-20T22:44:57.977Z 2 040-059 72 72 72 127 127 127 F F F
79412024-02-20T22:44:57.977Z 3 060-079 67 67 67 119 119 119 F F F
79422024-02-20T22:44:57.977Z 4 080-099 72 72 72 127 127 127 F F F
79432024-02-20T22:44:57.984Z 5 100-119 65 65 65 115 115 115 F F F
79442024-02-20T22:44:57.984Z 6 120-139 68 68 68 123 123 123 F F F
79452024-02-20T22:44:57.984Z 7 140-159 68 68 68 123 123 123 F F F
79462024-02-20T22:44:57.984Z 8 160-179 68 68 68 120 120 120 F F F
79472024-02-20T22:44:57.984Z 9 180-199 72 72 72 128 128 128 F F F
79482024-02-20T22:44:57.984Z 10 200-219 72 72 72 128 128 128 F F F
79492024-02-20T22:44:57.984Z 11 220-239 72 72 72 127 127 127 F F F
79502024-02-20T22:44:57.984Z 12 240-259 69 69 69 124 124 124 F F F
79512024-02-20T22:44:57.984Z 13 260-279 73 73 73 130 130 130 F F F
79522024-02-20T22:44:57.984Z 14 280-299 68 68 68 122 122 122 F F F
79532024-02-20T22:44:57.984Z 15 300-319 73 73 73 130 130 130 F F F
79542024-02-20T22:44:57.984Z 16 320-339 71 71 71 126 126 126 F F F
79552024-02-20T22:44:57.984Z 17 340-359 71 71 71 126 126 126 F F F
79562024-02-20T22:44:57.985Z 18 360-379 72 72 72 129 129 129 F F F
79572024-02-20T22:44:57.985Z 19 380-399 71 71 71 126 126 126 F F F
79582024-02-20T22:44:57.985Z 20 400-419 71 71 71 126 126 126 F F F
79592024-02-20T22:44:57.985Z 21 420-439 71 71 71 126 126 126 F F F
79602024-02-20T22:44:57.985Z 22 440-459 72 72 72 127 127 127 F F F
79612024-02-20T22:44:57.985Z 23 460-479 72 72 72 127 127 127 F F F
79622024-02-20T22:44:57.985Z 24 480-499 73 73 73 130 130 130 F F F
79632024-02-20T22:44:57.985Z 25 500-519 71 71 71 126 126 126 F F F
79642024-02-20T22:44:57.985Z 26 520-539 72 72 72 127 127 127 F F F
79652024-02-20T22:44:57.985Z 27 540-559 72 72 72 127 127 127 F F F
79662024-02-20T22:44:57.985Z 28 560-579 72 72 72 128 128 128 F F F
79672024-02-20T22:44:57.985Z 29 580-599 72 72 72 128 128 128 F F F
79682024-02-20T22:44:57.985ZMax gen: 73, Max flush: 130
79692024-02-20T22:44:57.985Z
79702024-02-20T22:44:57.985ZBegin loop 37
79712024-02-20T22:44:57.985ZDownstairs to restart: 1
79722024-02-20T22:44:58.002Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527126054Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434}
79732024-02-20T22:44:58.002Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527404964Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434}
79742024-02-20T22:44:58.002Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527432963Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434}
79752024-02-20T22:44:58.009Z{"msg":"Crucible 573aab49-588a-470b-adb3-c36bbf01d8c3 has session id: bfc28abf-6884-4726-86fd-5e287161a80b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527455642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79762024-02-20T22:44:58.009Z{"msg":"Upstairs opts: Upstairs UUID: 573aab49-588a-470b-adb3-c36bbf01d8c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.52750151Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79772024-02-20T22:44:58.009Z{"msg":"Crucible stats registered with UUID: 573aab49-588a-470b-adb3-c36bbf01d8c3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527529129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79782024-02-20T22:44:58.009ZCrucible runtime is spawned
79792024-02-20T22:44:58.010Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527668914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434}
79802024-02-20T22:44:58.010Z{"msg":"573aab49-588a-470b-adb3-c36bbf01d8c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.52779982Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79812024-02-20T22:44:58.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527827459Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79822024-02-20T22:44:58.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527845478Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79832024-02-20T22:44:58.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527875647Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79842024-02-20T22:44:58.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527893206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79852024-02-20T22:44:58.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527923555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79862024-02-20T22:44:58.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527939425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79872024-02-20T22:44:58.010Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527957344Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"io task","client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79882024-02-20T22:44:58.010Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.527988043Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"io task","client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79892024-02-20T22:44:58.010Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528029581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"io task","client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79902024-02-20T22:44:58.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.5280575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"io task","client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79912024-02-20T22:44:58.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528225994Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"io task","client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79922024-02-20T22:44:58.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528291512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"io task","client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79932024-02-20T22:44:58.010Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528779505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79942024-02-20T22:44:58.010Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528829773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79952024-02-20T22:44:58.010Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528873061Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79962024-02-20T22:44:58.010Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528943619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79972024-02-20T22:44:58.010Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.528982197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79982024-02-20T22:44:58.010Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.529007836Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
79992024-02-20T22:44:58.010Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.52946138Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80002024-02-20T22:44:58.010Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.529506888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80012024-02-20T22:44:58.010Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.529602835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80022024-02-20T22:44:58.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530072098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80032024-02-20T22:44:58.011Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530122666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80042024-02-20T22:44:58.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530142066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80052024-02-20T22:44:58.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530162645Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80062024-02-20T22:44:58.011Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530177924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80072024-02-20T22:44:58.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530191644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80082024-02-20T22:44:58.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530204314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80092024-02-20T22:44:58.011Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530218403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80102024-02-20T22:44:58.011Z{"msg":"[0]R flush_numbers[0..12]: [126, 126, 127, 119, 127, 115, 123, 123, 120, 128, 128, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530240162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80112024-02-20T22:44:58.011Z{"msg":"[0]R generation[0..12]: [71, 71, 72, 67, 72, 65, 68, 68, 68, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530254812Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80122024-02-20T22:44:58.011Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530275691Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80132024-02-20T22:44:58.011Z{"msg":"[1]R flush_numbers[0..12]: [126, 126, 127, 119, 127, 115, 123, 123, 120, 128, 128, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.5302901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80142024-02-20T22:44:58.011Z{"msg":"[1]R generation[0..12]: [71, 71, 72, 67, 72, 65, 68, 68, 68, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.53030472Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80152024-02-20T22:44:58.011Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530326189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80162024-02-20T22:44:58.011Z{"msg":"[2]R flush_numbers[0..12]: [126, 126, 127, 119, 127, 115, 123, 123, 120, 128, 128, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530347078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80172024-02-20T22:44:58.011ZWait for a query_work_queue command to finish before sending IO
80182024-02-20T22:44:58.011Z{"msg":"[2]R generation[0..12]: [71, 71, 72, 67, 72, 65, 68, 68, 68, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530377557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80192024-02-20T22:44:58.011Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530397927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80202024-02-20T22:44:58.011Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530414096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80212024-02-20T22:44:58.011Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530428126Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80222024-02-20T22:44:58.011Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530448935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80232024-02-20T22:44:58.011Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530462584Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80242024-02-20T22:44:58.011Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530482724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80252024-02-20T22:44:58.011Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530495143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80262024-02-20T22:44:58.011Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530515272Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80272024-02-20T22:44:58.012Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530529012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80282024-02-20T22:44:58.012Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530542451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80292024-02-20T22:44:58.012Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530554821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80302024-02-20T22:44:58.012Z{"msg":"573aab49-588a-470b-adb3-c36bbf01d8c3 is now active with session: bfc28abf-6884-4726-86fd-5e287161a80b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.53057429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80312024-02-20T22:44:58.012Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.53058674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434,"session_id":"bfc28abf-6884-4726-86fd-5e287161a80b"}
80322024-02-20T22:44:58.012Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:44:57.530605999Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434}
80332024-02-20T22:44:58.012ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80342024-02-20T22:44:58.012ZLoading write count information from file "/tmp/test_repair_verify.data"
80352024-02-20T22:44:58.012ZRun Repair workload
80362024-02-20T22:44:58.012Z01/30 Write block 290 len 40960 data: 6 7 6 5 4 3 4 4 6 6
80372024-02-20T22:44:58.012Z02/30 Write block 535 len 24576 data: 9 10 8 9 7 8
80382024-02-20T22:44:58.012Z03/30 Read block 330 len 12288
80392024-02-20T22:44:58.018Z04/30 Flush
80402024-02-20T22:44:58.018Z05/30 Write block 226 len 32768 data: 4 5 7 9 11 10 11 13
80412024-02-20T22:44:58.018Z06/30 Read block 563 len 12288
80422024-02-20T22:44:58.018Z07/30 Read block 453 len 16384
80432024-02-20T22:44:58.018Z08/30 Flush
80442024-02-20T22:44:58.018Z09/30 Write block 16 len 4096 data: 7
80452024-02-20T22:44:58.018Z10/30 Write block 2 len 16384 data: 5 5 6 7
80462024-02-20T22:44:58.018Z11/30 Write block 130 len 24576 data: 6 6 6 7 7 11
80472024-02-20T22:44:58.018Z12/30 Write block 198 len 8192 data: 7 8
80482024-02-20T22:44:58.024Z13/30 Write block 555 len 16384 data: 5 5 4 3
80492024-02-20T22:44:58.024Z14/30 Write block 160 len 12288 data: 6 5 4
80502024-02-20T22:44:58.024Z15/30 Write block 309 len 24576 data: 9 8 8 7 6 8
80512024-02-20T22:44:58.024Z16/30 Write block 300 len 24576 data: 5 9 10 10 10 10
80522024-02-20T22:44:58.024Z17/30 Write block 220 len 8192 data: 4 4
80532024-02-20T22:44:58.024Z18/30 Read block 564 len 40960
80542024-02-20T22:44:58.024Z19/30 Read block 143 len 36864
80552024-02-20T22:44:58.024Z20/30 Read block 231 len 20480
80562024-02-20T22:44:58.024Z21/30 Read block 512 len 28672
80572024-02-20T22:44:58.028Z22/30 Write block 18 len 36864 data: 6 5 5 4 5 5 5 10 9
80582024-02-20T22:44:58.028Z23/30 Flush
80592024-02-20T22:44:58.028Z24/30 Write block 77 len 28672 data: 4 4 4 4 5 4 4
80602024-02-20T22:44:58.032Z25/30 Read block 15 len 28672
80612024-02-20T22:44:58.032Z26/30 Write block 165 len 8192 data: 5 6
80622024-02-20T22:44:58.032Z27/30 Write block 364 len 4096 data: 12
80632024-02-20T22:44:58.032Z28/30 Read block 51 len 4096
80642024-02-20T22:44:58.032Z29/30 Write block 154 len 28672 data: 5 5 6 7 7 7 7
80652024-02-20T22:44:58.032Z30/30 Read block 339 len 8192
80662024-02-20T22:44:58.037Z----------------------------------------------------------------
80672024-02-20T22:44:58.037Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80682024-02-20T22:44:58.037ZStates: Active Active Active
80692024-02-20T22:44:58.037ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80702024-02-20T22:44:58.037Z 1 Acked 1000 Write 10 Done Sent Done false
80712024-02-20T22:44:58.037Z 2 Acked 1001 Write 6 Done Sent Done false
80722024-02-20T22:44:58.037Z 3 Acked 1002 Read 3 Done Sent Done false
80732024-02-20T22:44:58.037Z 4 Acked 1003 Flush 0 Done Sent Done false
80742024-02-20T22:44:58.037Z 5 Acked 1004 Write 8 Done Sent Done false
80752024-02-20T22:44:58.037Z 6 Acked 1005 Read 3 Done Sent Done false
80762024-02-20T22:44:58.037Z 7 Acked 1006 Read 4 Done Sent Done false
80772024-02-20T22:44:58.037Z 8 Acked 1007 Flush 0 Done Sent Done false
80782024-02-20T22:44:58.037Z 9 Acked 1008 Write 1 Done Sent Done false
80792024-02-20T22:44:58.037Z 10 Acked 1009 Write 4 Done Sent Done false
80802024-02-20T22:44:58.037Z 11 Acked 1010 Write 6 Done Sent Done false
80812024-02-20T22:44:58.037Z 12 Acked 1011 Write 2 Done Sent Done false
80822024-02-20T22:44:58.037Z 13 Acked 1012 Write 4 Done Sent Done false
80832024-02-20T22:44:58.038Z 14 Acked 1013 Write 3 Done Sent Done false
80842024-02-20T22:44:58.038Z 15 Acked 1014 Write 6 Done Sent Done false
80852024-02-20T22:44:58.038Z 16 Acked 1015 Write 6 Done Sent Done false
80862024-02-20T22:44:58.038Z 17 Acked 1016 Write 2 Done Sent Done false
80872024-02-20T22:44:58.038Z 18 Acked 1017 Read 10 Done Sent Done false
80882024-02-20T22:44:58.038Z 19 Acked 1018 Read 9 Done Sent Done false
80892024-02-20T22:44:58.038Z 20 Acked 1019 Read 5 Done Sent Done false
80902024-02-20T22:44:58.038Z 21 Acked 1020 Read 7 Done Sent Done false
80912024-02-20T22:44:58.038Z 22 Acked 1021 Write 9 Done Sent Done false
80922024-02-20T22:44:58.038Z 23 Acked 1022 Flush 0 Done Sent Done false
80932024-02-20T22:44:58.038Z 24 Acked 1023 Write 7 Done Sent Done false
80942024-02-20T22:44:58.038Z 25 Acked 1024 Read 7 Done Sent Done false
80952024-02-20T22:44:58.038Z 26 Acked 1025 Write 2 Done Sent Done false
80962024-02-20T22:44:58.038Z 27 Acked 1026 Write 1 Done Sent Done false
80972024-02-20T22:44:58.038Z 28 Acked 1027 Read 1 Done Sent Done false
80982024-02-20T22:44:58.038Z 29 Acked 1028 Write 7 Done Sent Done false
80992024-02-20T22:44:58.038Z 30 Acked 1029 Read 2 Done Sent Done false
81002024-02-20T22:44:58.038Z STATES DS:0 DS:1 DS:2 TOTAL
81012024-02-20T22:44:58.038Z New 0 0 0 0
81022024-02-20T22:44:58.038Z Sent 0 30 0 30
81032024-02-20T22:44:58.038Z Done 30 0 30 60
81042024-02-20T22:44:58.038Z Skipped 0 0 0 0
81052024-02-20T22:44:58.038Z Error 0 0 0 0
81062024-02-20T22:44:58.038ZLast Flush: 1022 0 1022
81072024-02-20T22:44:58.038ZDownstairs last five completed:
81082024-02-20T22:44:58.039ZUpstairs last five completed: 30 29 28 27 26
81092024-02-20T22:44:58.039Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:44:57.560843007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1434}
81102024-02-20T22:44:58.039ZWrote out file "/tmp/test_repair_verify.data"
81112024-02-20T22:44:58.044Z
81122024-02-20T22:44:58.044ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81132024-02-20T22:44:58.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81142024-02-20T22:44:58.057Z 0 000-019 74 71 74 133 126 133 F F F <---
81152024-02-20T22:44:58.057Z 1 020-039 74 71 74 133 126 133 F F F <---
81162024-02-20T22:44:58.058Z 2 040-059 72 72 72 127 127 127 F F F
81172024-02-20T22:44:58.058Z 3 060-079 67 67 67 119 119 119 T F T <---
81182024-02-20T22:44:58.058Z 4 080-099 72 72 72 127 127 127 T F T <---
81192024-02-20T22:44:58.616Z 5 100-119 65 65 65 115 115 115 F F F
81202024-02-20T22:44:58.616Z 6 120-139 74 68 74 133 123 133 F F F <---
81212024-02-20T22:44:58.616Z 7 140-159 68 68 68 123 123 123 T F T <---
81222024-02-20T22:44:58.616Z 8 160-179 74 68 74 133 120 133 T F T <---
81232024-02-20T22:44:58.616Z 9 180-199 74 72 74 133 128 133 F F F <---
81242024-02-20T22:44:58.616Z 10 200-219 72 72 72 128 128 128 F F F
81252024-02-20T22:44:58.616Z 11 220-239 74 72 74 133 127 133 F F F <---
81262024-02-20T22:44:58.616Z 12 240-259 69 69 69 124 124 124 F F F
81272024-02-20T22:44:58.616Z 13 260-279 73 73 73 130 130 130 F F F
81282024-02-20T22:44:58.616Z 14 280-299 74 68 74 131 122 131 F F F <---
81292024-02-20T22:44:58.616Z 15 300-319 74 73 74 133 130 133 F F F <---
81302024-02-20T22:44:58.616Z 16 320-339 71 71 71 126 126 126 F F F
81312024-02-20T22:44:58.616Z 17 340-359 71 71 71 126 126 126 F F F
81322024-02-20T22:44:58.616Z 18 360-379 72 72 72 129 129 129 T F T <---
81332024-02-20T22:44:58.616Z 19 380-399 71 71 71 126 126 126 F F F
81342024-02-20T22:44:58.616Z 20 400-419 71 71 71 126 126 126 F F F
81352024-02-20T22:44:58.616Z 21 420-439 71 71 71 126 126 126 F F F
81362024-02-20T22:44:58.616Z 22 440-459 72 72 72 127 127 127 F F F
81372024-02-20T22:44:58.616Z 23 460-479 72 72 72 127 127 127 F F F
81382024-02-20T22:44:58.616Z 24 480-499 73 73 73 130 130 130 F F F
81392024-02-20T22:44:58.616Z 25 500-519 71 71 71 126 126 126 F F F
81402024-02-20T22:44:58.616Z 26 520-539 74 72 74 131 127 131 F F F <---
81412024-02-20T22:44:58.617Z 27 540-559 74 72 74 133 127 133 F F F <---
81422024-02-20T22:44:58.617Z 28 560-579 72 72 72 128 128 128 F F F
81432024-02-20T22:44:58.617Z 29 580-599 72 72 72 128 128 128 F F F
81442024-02-20T22:44:58.617ZMax gen: 74, Max flush: 133
81452024-02-20T22:44:58.617ZError: Difference in extent metadata found!
81462024-02-20T22:44:58.617ZA Difference in extent metadata is expected here
81472024-02-20T22:44:58.617ZOn loop 37
81482024-02-20T22:44:58.617Z
81492024-02-20T22:44:58.617ZVerifying data now
81502024-02-20T22:45:02.730ZNo change after verify
81512024-02-20T22:45:02.730ZLoop: 37 Downstairs dump after verify (and repair):
81522024-02-20T22:45:02.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81532024-02-20T22:45:02.744Z 0 000-019 74 74 74 133 133 133 F F F
81542024-02-20T22:45:02.744Z 1 020-039 74 74 74 133 133 133 F F F
81552024-02-20T22:45:02.744Z 2 040-059 72 72 72 127 127 127 F F F
81562024-02-20T22:45:02.744Z 3 060-079 75 75 75 134 134 134 F F F
81572024-02-20T22:45:02.744Z 4 080-099 75 75 75 134 134 134 F F F
81582024-02-20T22:45:02.744Z 5 100-119 65 65 65 115 115 115 F F F
81592024-02-20T22:45:02.744Z 6 120-139 74 74 74 133 133 133 F F F
81602024-02-20T22:45:02.744Z 7 140-159 75 75 75 134 134 134 F F F
81612024-02-20T22:45:02.744Z 8 160-179 75 75 75 134 134 134 F F F
81622024-02-20T22:45:02.745Z 9 180-199 74 74 74 133 133 133 F F F
81632024-02-20T22:45:02.758Z 10 200-219 72 72 72 128 128 128 F F F
81642024-02-20T22:45:02.758Z 11 220-239 74 74 74 133 133 133 F F F
81652024-02-20T22:45:02.758Z 12 240-259 69 69 69 124 124 124 F F F
81662024-02-20T22:45:02.758Z 13 260-279 73 73 73 130 130 130 F F F
81672024-02-20T22:45:02.758Z 14 280-299 74 74 74 131 131 131 F F F
81682024-02-20T22:45:02.758Z 15 300-319 74 74 74 133 133 133 F F F
81692024-02-20T22:45:02.758Z 16 320-339 71 71 71 126 126 126 F F F
81702024-02-20T22:45:02.758Z 17 340-359 71 71 71 126 126 126 F F F
81712024-02-20T22:45:02.759Z 18 360-379 75 75 75 134 134 134 F F F
81722024-02-20T22:45:02.759Z 19 380-399 71 71 71 126 126 126 F F F
81732024-02-20T22:45:02.759Z 20 400-419 71 71 71 126 126 126 F F F
81742024-02-20T22:45:02.759Z 21 420-439 71 71 71 126 126 126 F F F
81752024-02-20T22:45:02.759Z 22 440-459 72 72 72 127 127 127 F F F
81762024-02-20T22:45:02.759Z 23 460-479 72 72 72 127 127 127 F F F
81772024-02-20T22:45:02.759Z 24 480-499 73 73 73 130 130 130 F F F
81782024-02-20T22:45:02.760Z 25 500-519 71 71 71 126 126 126 F F F
81792024-02-20T22:45:02.760Z 26 520-539 74 74 74 131 131 131 F F F
81802024-02-20T22:45:02.760Z 27 540-559 74 74 74 133 133 133 F F F
81812024-02-20T22:45:02.760Z 28 560-579 72 72 72 128 128 128 F F F
81822024-02-20T22:45:02.760Z 29 580-599 72 72 72 128 128 128 F F F
81832024-02-20T22:45:02.760ZMax gen: 75, Max flush: 134
81842024-02-20T22:45:02.760Z
81852024-02-20T22:45:02.760ZBegin loop 38
81862024-02-20T22:45:02.761ZDownstairs to restart: 2
81872024-02-20T22:45:02.769Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.292581593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442}
81882024-02-20T22:45:02.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.292868573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442}
81892024-02-20T22:45:02.769Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.292906321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442}
81902024-02-20T22:45:02.769Z{"msg":"Crucible 9d1d6945-fa72-4afa-85d4-a1264a086645 has session id: b3d21eeb-6952-43cd-bdee-f735bc937a69","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29293268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81912024-02-20T22:45:02.769Z{"msg":"Upstairs opts: Upstairs UUID: 9d1d6945-fa72-4afa-85d4-a1264a086645, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.292955099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81922024-02-20T22:45:02.769Z{"msg":"Crucible stats registered with UUID: 9d1d6945-fa72-4afa-85d4-a1264a086645","v":0,"name":"crucible","level":30Crucible runtime is spawned
81932024-02-20T22:45:02.769Z,"time":"2024-02-20T22:45:02.292977809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81942024-02-20T22:45:02.769Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293132403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442}
81952024-02-20T22:45:02.769Z{"msg":"9d1d6945-fa72-4afa-85d4-a1264a086645 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293171972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81962024-02-20T22:45:02.769Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293190531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81972024-02-20T22:45:02.769Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29321618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81982024-02-20T22:45:02.769Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293251839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
81992024-02-20T22:45:02.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293279828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82002024-02-20T22:45:02.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293320606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82012024-02-20T22:45:02.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293348636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82022024-02-20T22:45:02.770Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293402103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82032024-02-20T22:45:02.770Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293533659Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82042024-02-20T22:45:02.770Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293572777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82052024-02-20T22:45:02.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293589007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82062024-02-20T22:45:02.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293604666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82072024-02-20T22:45:02.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.293627306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82082024-02-20T22:45:02.779Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294245273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82092024-02-20T22:45:02.779Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294292142Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82102024-02-20T22:45:02.779Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294311051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82112024-02-20T22:45:02.779Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29433003Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82122024-02-20T22:45:02.779Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294365629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82132024-02-20T22:45:02.779Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294559582Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82142024-02-20T22:45:02.779Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294930979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82152024-02-20T22:45:02.779Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.294973507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82162024-02-20T22:45:02.779Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29517631Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82172024-02-20T22:45:02.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295475209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82182024-02-20T22:45:02.780Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295526298Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82192024-02-20T22:45:02.780Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295554246Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82202024-02-20T22:45:02.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295588375Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82212024-02-20T22:45:02.780Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295615854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82222024-02-20T22:45:02.780Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295641543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82232024-02-20T22:45:02.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295655453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82242024-02-20T22:45:02.780Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295676382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82252024-02-20T22:45:02.780Z{"msg":"[0]R flush_numbers[0..12]: [133, 133, 127, 134, 134, 115, 133, 134, 134, 133, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295691042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82262024-02-20T22:45:02.780Z{"msg":"[0]R generation[0..12]: [74, 74, 72, 75, 75, 65, 74, 75, 75, 74, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295713281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82272024-02-20T22:45:02.780Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29572789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82282024-02-20T22:45:02.780Z{"msg":"[1]R flush_numbers[0..12]: [133, 133, 127, 134, 134, 115, 133, 134, 134, 133, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29575021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82292024-02-20T22:45:02.780Z{"msg":"[1]R generation[0..12]: [74, 74, 72, 75, 75, 65, 74, 75, 75, 74, 72, 74]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
82302024-02-20T22:45:02.780Z,"time":"2024-02-20T22:45:02.295764349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82312024-02-20T22:45:02.780Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295791698Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82322024-02-20T22:45:02.780Z{"msg":"[2]R flush_numbers[0..12]: [133, 133, 127, 134, 134, 115, 133, 134, 134, 133, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295806448Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82332024-02-20T22:45:02.780Z{"msg":"[2]R generation[0..12]: [74, 74, 72, 75, 75, 65, 74, 75, 75, 74, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295827317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82342024-02-20T22:45:02.780Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295842306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82352024-02-20T22:45:02.780Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295863695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82362024-02-20T22:45:02.780Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295877665Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82372024-02-20T22:45:02.780Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295897734Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82382024-02-20T22:45:02.780Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295911244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82392024-02-20T22:45:02.780Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295931673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82402024-02-20T22:45:02.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295944573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82412024-02-20T22:45:02.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295958722Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82422024-02-20T22:45:02.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295978701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82432024-02-20T22:45:02.781Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.295992111Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82442024-02-20T22:45:02.781Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29600447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82452024-02-20T22:45:02.781Z{"msg":"9d1d6945-fa72-4afa-85d4-a1264a086645 is now active with session: b3d21eeb-6952-43cd-bdee-f735bc937a69","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.29601736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82462024-02-20T22:45:02.781Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.296036339Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442,"session_id":"b3d21eeb-6952-43cd-bdee-f735bc937a69"}
82472024-02-20T22:45:02.781Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:02.296048909Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442}
82482024-02-20T22:45:02.781ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82492024-02-20T22:45:02.781ZLoading write count information from file "/tmp/test_repair_verify.data"
82502024-02-20T22:45:02.781ZRun Repair workload
82512024-02-20T22:45:02.781Z01/30 Write block 247 len 4096 data: 6
82522024-02-20T22:45:02.781Z02/30 Write block 525 len 28672 data: 4 7 8 7 8 9 7
82532024-02-20T22:45:02.781Z03/30 Read block 458 len 16384
82542024-02-20T22:45:02.781Z04/30 Write block 316 len 32768 data: 9 10 8 9 10 8 10 6
82552024-02-20T22:45:02.781Z05/30 Write block 458 len 16384 data: 13 11 10 11
82562024-02-20T22:45:02.781Z06/30 Read block 90 len 24576
82572024-02-20T22:45:02.785Z07/30 Flush
82582024-02-20T22:45:02.785Z08/30 Flush
82592024-02-20T22:45:02.786Z09/30 Flush
82602024-02-20T22:45:02.786Z10/30 Write block 253 len 8192 data: 5 4
82612024-02-20T22:45:02.786Z11/30 Read block 539 len 28672
82622024-02-20T22:45:02.786Z12/30 Write block 54 len 24576 data: 7 7 5 4 5 6
82632024-02-20T22:45:02.786Z13/30 Read block 410 len 8192
82642024-02-20T22:45:02.786Z14/30 Write block 67 len 20480 data: 6 7 8 7 7
82652024-02-20T22:45:02.786Z15/30 Read block 232 len 12288
82662024-02-20T22:45:02.786Z16/30 Read block 320 len 20480
82672024-02-20T22:45:02.790Z17/30 Read block 184 len 20480
82682024-02-20T22:45:02.790Z18/30 Read block 66 len 8192
82692024-02-20T22:45:02.790Z19/30 Write block 63 len 4096 data: 3
82702024-02-20T22:45:02.790Z20/30 Read block 473 len 4096
82712024-02-20T22:45:02.790Z21/30 Write block 255 len 20480 data: 2 2 3 4 4
82722024-02-20T22:45:02.790Z22/30 Read block 245 len 32768
82732024-02-20T22:45:02.794Z23/30 Read block 62 len 16384
82742024-02-20T22:45:02.794Z24/30 Read block 59 len 32768
82752024-02-20T22:45:02.794Z25/30 Read block 484 len 24576
82762024-02-20T22:45:02.794Z26/30 Read block 426 len 24576
82772024-02-20T22:45:02.794Z27/30 Read block 230 len 36864
82782024-02-20T22:45:02.799Z28/30 Write block 558 len 16384 data: 4 3 3 5
82792024-02-20T22:45:02.799Z29/30 Read block 545 len 12288
82802024-02-20T22:45:02.799Z30/30 Write block 297 len 32768 data: 5 7 7 6 10 11 11 11
82812024-02-20T22:45:02.799Z----------------------------------------------------------------
82822024-02-20T22:45:02.799Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:21
82832024-02-20T22:45:02.799ZStates: Active Active Active
82842024-02-20T22:45:02.799ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82852024-02-20T22:45:02.799Z 10 Acked 1009 Write 2 Done Done Done false
82862024-02-20T22:45:02.799Z 11 Acked 1010 Read 7 Done Done Done false
82872024-02-20T22:45:02.799Z 12 Acked 1011 Write 6 Done Done Done false
82882024-02-20T22:45:02.799Z 13 Acked 1012 Read 2 Done Done Done false
82892024-02-20T22:45:02.799Z 14 Acked 1013 Write 5 Done Done Done false
82902024-02-20T22:45:02.799Z 15 Acked 1014 Read 3 Done Done Done false
82912024-02-20T22:45:02.799Z 16 Acked 1015 Read 5 Done Done Done false
82922024-02-20T22:45:02.800Z 17 Acked 1016 Read 5 Done Done Done false
82932024-02-20T22:45:02.800Z 18 Acked 1017 Read 2 Done Done Sent false
82942024-02-20T22:45:02.800Z 19 Acked 1018 Write 1 Done Done Sent false
82952024-02-20T22:45:02.800Z 20 Acked 1019 Read 1 Done Done Sent false
82962024-02-20T22:45:02.800Z 21 Acked 1020 Write 5 Done Done Sent false
82972024-02-20T22:45:02.800Z 22 Acked 1021 Read 8 Done Done Sent false
82982024-02-20T22:45:02.800Z 23 Acked 1022 Read 4 Done Done Sent false
82992024-02-20T22:45:02.800Z 24 Acked 1023 Read 8 Done Done Sent false
83002024-02-20T22:45:02.800Z 25 Acked 1024 Read 6 Done Done Sent false
83012024-02-20T22:45:02.800Z 26 Acked 1025 Read 6 Done Done Sent false
83022024-02-20T22:45:02.800Z 27 Acked 1026 Read 9 Done Done Sent false
83032024-02-20T22:45:02.800Z 28 Acked 1027 Write 4 Done Done Sent false
83042024-02-20T22:45:02.800Z 29 Acked 1028 Read 3 Done Done Sent false
83052024-02-20T22:45:02.800Z 30 Acked 1029 Write 8 Sent Sent Sent false
83062024-02-20T22:45:02.800Z STATES DS:0 DS:1 DS:2 TOTAL
83072024-02-20T22:45:02.800Z New 0 0 0 0
83082024-02-20T22:45:02.800Z Sent 1 1 13 15
83092024-02-20T22:45:02.800Z Done 20 20 8 48
83102024-02-20T22:45:02.800Z Skipped 0 0 0 0
83112024-02-20T22:45:02.800Z Error 0 0 0 0
83122024-02-20T22:45:02.800ZLast Flush: 1008 1008 1008
83132024-02-20T22:45:02.800ZDownstairs last five completed: 1008 1007 1006 1005 1004
83142024-02-20T22:45:02.800ZUpstairs last five completed: 30 29 28 27 26
83152024-02-20T22:45:02.800Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:02.324752712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1442}
83162024-02-20T22:45:02.800ZWrote out file "/tmp/test_repair_verify.data"
83172024-02-20T22:45:02.804Z
83182024-02-20T22:45:02.814ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83192024-02-20T22:45:02.823ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83202024-02-20T22:45:02.823Z 0 000-019 74 74 74 133 133 133 F F F
83212024-02-20T22:45:02.823Z 1 020-039 74 74 74 133 133 133 F F F
83222024-02-20T22:45:02.823Z 2 040-059 72 72 72 127 127 127 T T T
83232024-02-20T22:45:02.823Z 3 060-079 75 75 75 134 134 134 T T T
83242024-02-20T22:45:02.823Z 4 080-099 75 75 75 134 134 134 F F F
83252024-02-20T22:45:02.823Z 5 100-119 65 65 65 115 115 115 F F F
83262024-02-20T22:45:02.823Z 6 120-139 74 74 74 133 133 133 F F F
83272024-02-20T22:45:02.823Z 7 140-159 75 75 75 134 134 134 F F F
83282024-02-20T22:45:02.823Z 8 160-179 75 75 75 134 134 134 F F F
83292024-02-20T22:45:02.823Z 9 180-199 74 74 74 133 133 133 F F F
83302024-02-20T22:45:02.823Z 10 200-219 72 72 72 128 128 128 F F F
83312024-02-20T22:45:02.839Z 11 220-239 74 74 74 133 133 133 F F F
83322024-02-20T22:45:02.839Z 12 240-259 76 76 76 135 135 135 T T T
83332024-02-20T22:45:02.840Z 13 260-279 73 73 73 130 130 130 F F F
83342024-02-20T22:45:02.840Z 14 280-299 74 74 74 131 131 131 T T F <---
83352024-02-20T22:45:02.840Z 15 300-319 76 76 76 135 135 135 T T F <---
83362024-02-20T22:45:02.840Z 16 320-339 76 76 76 135 135 135 F F F
83372024-02-20T22:45:02.840Z 17 340-359 71 71 71 126 126 126 F F F
83382024-02-20T22:45:02.840Z 18 360-379 75 75 75 134 134 134 F F F
83392024-02-20T22:45:02.840Z 19 380-399 71 71 71 126 126 126 F F F
83402024-02-20T22:45:02.840Z 20 400-419 71 71 71 126 126 126 F F F
83412024-02-20T22:45:02.840Z 21 420-439 71 71 71 126 126 126 F F F
83422024-02-20T22:45:02.840Z 22 440-459 76 76 76 135 135 135 F F F
83432024-02-20T22:45:02.840Z 23 460-479 76 76 76 135 135 135 F F F
83442024-02-20T22:45:02.840Z 24 480-499 73 73 73 130 130 130 F F F
83452024-02-20T22:45:02.840Z 25 500-519 71 71 71 126 126 126 F F F
83462024-02-20T22:45:02.840Z 26 520-539 76 76 76 135 135 135 F F F
83472024-02-20T22:45:02.840Z 27 540-559 74 74 74 133 133 133 T T F <---
83482024-02-20T22:45:02.840Z 28 560-579 72 72 72 128 128 128 T T F <---
83492024-02-20T22:45:02.840Z 29 580-599 72 72 72 128 128 128 F F F
83502024-02-20T22:45:02.840ZMax gen: 76, Max flush: 135
83512024-02-20T22:45:02.840ZError: Difference in extent metadata found!
83522024-02-20T22:45:02.840ZA Difference in extent metadata is expected here
83532024-02-20T22:45:02.840ZOn loop 38
83542024-02-20T22:45:02.840Z
83552024-02-20T22:45:02.840ZVerifying data now
83562024-02-20T22:45:08.228ZNo change after verify
83572024-02-20T22:45:08.228ZLoop: 38 Downstairs dump after verify (and repair):
83582024-02-20T22:45:08.243ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83592024-02-20T22:45:08.243Z 0 000-019 74 74 74 133 133 133 F F F
83602024-02-20T22:45:08.244Z 1 020-039 74 74 74 133 133 133 F F F
83612024-02-20T22:45:08.244Z 2 040-059 77 77 77 136 136 136 F F F
83622024-02-20T22:45:08.244Z 3 060-079 77 77 77 136 136 136 F F F
83632024-02-20T22:45:08.244Z 4 080-099 75 75 75 134 134 134 F F F
83642024-02-20T22:45:08.244Z 5 100-119 65 65 65 115 115 115 F F F
83652024-02-20T22:45:08.244Z 6 120-139 74 74 74 133 133 133 F F F
83662024-02-20T22:45:08.244Z 7 140-159 75 75 75 134 134 134 F F F
83672024-02-20T22:45:08.244Z 8 160-179 75 75 75 134 134 134 F F F
83682024-02-20T22:45:08.244Z 9 180-199 74 74 74 133 133 133 F F F
83692024-02-20T22:45:08.244Z 10 200-219 72 72 72 128 128 128 F F F
83702024-02-20T22:45:08.244Z 11 220-239 74 74 74 133 133 133 F F F
83712024-02-20T22:45:08.244Z 12 240-259 77 77 77 136 136 136 F F F
83722024-02-20T22:45:08.244Z 13 260-279 73 73 73 130 130 130 F F F
83732024-02-20T22:45:08.244Z 14 280-299 77 77 77 136 136 136 F F F
83742024-02-20T22:45:08.249Z 15 300-319 77 77 77 136 136 136 F F F
83752024-02-20T22:45:08.249Z 16 320-339 76 76 76 135 135 135 F F F
83762024-02-20T22:45:08.249Z 17 340-359 71 71 71 126 126 126 F F F
83772024-02-20T22:45:08.249Z 18 360-379 75 75 75 134 134 134 F F F
83782024-02-20T22:45:08.249Z 19 380-399 71 71 71 126 126 126 F F F
83792024-02-20T22:45:08.249Z 20 400-419 71 71 71 126 126 126 F F F
83802024-02-20T22:45:08.249Z 21 420-439 71 71 71 126 126 126 F F F
83812024-02-20T22:45:08.249Z 22 440-459 76 76 76 135 135 135 F F F
83822024-02-20T22:45:08.249Z 23 460-479 76 76 76 135 135 135 F F F
83832024-02-20T22:45:08.250Z 24 480-499 73 73 73 130 130 130 F F F
83842024-02-20T22:45:08.250Z 25 500-519 71 71 71 126 126 126 F F F
83852024-02-20T22:45:08.250Z 26 520-539 76 76 76 135 135 135 F F F
83862024-02-20T22:45:08.250Z 27 540-559 77 77 77 136 136 136 F F F
83872024-02-20T22:45:08.250Z 28 560-579 77 77 77 136 136 136 F F F
83882024-02-20T22:45:08.250Z 29 580-599 72 72 72 128 128 128 F F F
83892024-02-20T22:45:08.250ZMax gen: 77, Max flush: 136
83902024-02-20T22:45:08.250Z
83912024-02-20T22:45:08.250ZBegin loop 39
83922024-02-20T22:45:08.250ZDownstairs to restart: 2
83932024-02-20T22:45:08.265Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.791510657Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450}
83942024-02-20T22:45:08.265Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.791811436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450}
83952024-02-20T22:45:08.274Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.791844495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450}
83962024-02-20T22:45:08.275ZCrucible runtime is spawned
83972024-02-20T22:45:08.275Z{"msg":"Crucible f8eff149-e0ca-4309-8b82-783c686b0237 has session id: 5d3f67ec-3bb7-41d5-b073-542bdc822300","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.791886213Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
83982024-02-20T22:45:08.275Z{"msg":"Upstairs opts: Upstairs UUID: f8eff149-e0ca-4309-8b82-783c686b0237, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.791964421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
83992024-02-20T22:45:08.275Z{"msg":"Crucible stats registered with UUID: f8eff149-e0ca-4309-8b82-783c686b0237","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.791992129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84002024-02-20T22:45:08.275Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792014399Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450}
84012024-02-20T22:45:08.275Z{"msg":"f8eff149-e0ca-4309-8b82-783c686b0237 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792140874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84022024-02-20T22:45:08.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792215312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84032024-02-20T22:45:08.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792279419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84042024-02-20T22:45:08.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792316558Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84052024-02-20T22:45:08.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792344147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84062024-02-20T22:45:08.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792371496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84072024-02-20T22:45:08.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792416194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84082024-02-20T22:45:08.275Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792459983Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"io task","client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84092024-02-20T22:45:08.275Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792500841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"io task","client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84102024-02-20T22:45:08.275Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.79253991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"io task","client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84112024-02-20T22:45:08.275Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792575439Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"io task","client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84122024-02-20T22:45:08.275Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792619987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"io task","client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84132024-02-20T22:45:08.275Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.792645256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"io task","client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84142024-02-20T22:45:08.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793221125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84152024-02-20T22:45:08.275Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793257204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84162024-02-20T22:45:08.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793272854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84172024-02-20T22:45:08.275Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793296673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84182024-02-20T22:45:08.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793325992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84192024-02-20T22:45:08.276Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793363671Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84202024-02-20T22:45:08.276Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793837894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84212024-02-20T22:45:08.276Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793887642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84222024-02-20T22:45:08.276Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.793962419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84232024-02-20T22:45:08.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794342056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84242024-02-20T22:45:08.276Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794415783Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84252024-02-20T22:45:08.276Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794452992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84262024-02-20T22:45:08.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794467621Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84272024-02-20T22:45:08.276Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794482751Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84282024-02-20T22:45:08.276Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.79450367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84292024-02-20T22:45:08.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794516999Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84302024-02-20T22:45:08.276Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794537129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84312024-02-20T22:45:08.276Z{"msg":"[0]R flush_numbers[0..12]: [133, 133, 136, 136, 134, 115, 133, 134, 134, 133, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794551868Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84322024-02-20T22:45:08.276Z{"msg":"[0]R generation[0..12]: [74, 74, 77, 77, 75, 65, 74, 75, 75, 74, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794566648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84332024-02-20T22:45:08.276Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794580857Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84342024-02-20T22:45:08.276Z{"msg":"[1]R flush_numbers[0..12]: [133, 133, 136, 136, 134, 115, 133, 134, 134, 133, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794602176Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84352024-02-20T22:45:08.276Z{"msg":"[1]R generation[0..12]: [74, 74, 77, 77, 75, 65, 74, 75, 75, 74, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794616706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84362024-02-20T22:45:08.276Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794650585Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84372024-02-20T22:45:08.276Z{"msg":"[2]R flush_numbers[0..12]: [133, 133, 136, 136, 134, 115, 133, 134, 134, 133, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794666704Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84382024-02-20T22:45:08.276Z{"msg":"[2]R generation[0..12]: [74, 74, 77, 77, 75, 65, 74, 75, 75, 74, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794680393Z","hostnameWait for a query_work_queue command to finish before sending IO
84392024-02-20T22:45:08.276Z":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84402024-02-20T22:45:08.277Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794706272Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84412024-02-20T22:45:08.277Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794720912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84422024-02-20T22:45:08.277Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794741941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84432024-02-20T22:45:08.277Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794755501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84442024-02-20T22:45:08.277Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.79477789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84452024-02-20T22:45:08.277Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794794899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84462024-02-20T22:45:08.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794814268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84472024-02-20T22:45:08.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794828098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84482024-02-20T22:45:08.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794853667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84492024-02-20T22:45:08.277Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794873206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84502024-02-20T22:45:08.277Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794893476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84512024-02-20T22:45:08.277Z{"msg":"f8eff149-e0ca-4309-8b82-783c686b0237 is now active with session: 5d3f67ec-3bb7-41d5-b073-542bdc822300","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794912755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84522024-02-20T22:45:08.277Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794932784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450,"session_id":"5d3f67ec-3bb7-41d5-b073-542bdc822300"}
84532024-02-20T22:45:08.277Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:07.794945464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450}
84542024-02-20T22:45:08.277ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84552024-02-20T22:45:08.277ZLoading write count information from file "/tmp/test_repair_verify.data"
84562024-02-20T22:45:08.277ZRun Repair workload
84572024-02-20T22:45:08.277Z01/30 Write block 477 len 28672 data: 12 12 9 11 11 10 10
84582024-02-20T22:45:08.282Z02/30 Read block 24 len 20480
84592024-02-20T22:45:08.282Z03/30 Write block 400 len 4096 data: 7
84602024-02-20T22:45:08.283Z04/30 Write block 381 len 4096 data: 3
84612024-02-20T22:45:08.283Z05/30 Read block 395 len 16384
84622024-02-20T22:45:08.283Z06/30 Write block 170 len 24576 data: 8 7 8 8 5 7
84632024-02-20T22:45:08.283Z07/30 Read block 77 len 8192
84642024-02-20T22:45:08.283Z08/30 Read block 307 len 12288
84652024-02-20T22:45:08.283Z09/30 Read block 279 len 4096
84662024-02-20T22:45:08.283Z10/30 Read block 578 len 24576
84672024-02-20T22:45:08.283Z11/30 Read block 121 len 24576
84682024-02-20T22:45:08.283Z12/30 Read block 38 len 28672
84692024-02-20T22:45:08.283Z13/30 Read block 559 len 20480
84702024-02-20T22:45:08.283Z14/30 Read block 497 len 24576
84712024-02-20T22:45:08.283Z15/30 Write block 499 len 28672 data: 9 7 6 5 4 6 7
84722024-02-20T22:45:08.283Z16/30 Read block 334 len 40960
84732024-02-20T22:45:08.287Z17/30 Write block 578 len 32768 data: 7 9 9 8 9 8 6 6
84742024-02-20T22:45:08.287Z18/30 Write block 565 len 20480 data: 5 6 6 6 5
84752024-02-20T22:45:08.287Z19/30 Write block 530 len 16384 data: 10 8 5 6
84762024-02-20T22:45:08.287Z20/30 Write block 468 len 36864 data: 11 12 8 7 8 8 9 9 8
84772024-02-20T22:45:08.287Z21/30 Flush
84782024-02-20T22:45:08.292Z22/30 Write block 334 len 40960 data: 9 7 7 7 6 7 6 7 7 7
84792024-02-20T22:45:08.292Z23/30 Write block 204 len 16384 data: 5 6 6 6
84802024-02-20T22:45:08.292Z24/30 Read block 401 len 8192
84812024-02-20T22:45:08.292Z25/30 Write block 174 len 36864 data: 6 8 7 7 7 9 9 8 8
84822024-02-20T22:45:08.292Z26/30 Flush
84832024-02-20T22:45:08.298Z27/30 Write block 348 len 4096 data: 8
84842024-02-20T22:45:08.298Z28/30 Read block 72 len 4096
84852024-02-20T22:45:08.298Z29/30 Read block 518 len 20480
84862024-02-20T22:45:08.298Z30/30 Read block 552 len 32768
84872024-02-20T22:45:08.298Z----------------------------------------------------------------
84882024-02-20T22:45:08.298Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84892024-02-20T22:45:08.298ZStates: Active Active Active
84902024-02-20T22:45:08.298ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84912024-02-20T22:45:08.298Z 1 Acked 1000 Write 7 Done Done Done false
84922024-02-20T22:45:08.298Z 2 Acked 1001 Read 5 Done Done Done false
84932024-02-20T22:45:08.298Z 3 Acked 1002 Write 1 Done Done Sent false
84942024-02-20T22:45:08.298Z 4 Acked 1003 Write 1 Done Done Sent false
84952024-02-20T22:45:08.298Z 5 Acked 1004 Read 4 Done Done Sent false
84962024-02-20T22:45:08.298Z 6 Acked 1005 Write 6 Done Done Sent false
84972024-02-20T22:45:08.298Z 7 Acked 1006 Read 2 Done Done Sent false
84982024-02-20T22:45:08.299Z 8 Acked 1007 Read 3 Done Done Sent false
84992024-02-20T22:45:08.299Z 9 Acked 1008 Read 1 Done Done Sent false
85002024-02-20T22:45:08.299Z 10 Acked 1009 Read 6 Done Done Sent false
85012024-02-20T22:45:08.299Z 11 Acked 1010 Read 6 Done Done Sent false
85022024-02-20T22:45:08.299Z 12 Acked 1011 Read 7 Done Done Sent false
85032024-02-20T22:45:08.299Z 13 Acked 1012 Read 5 Done Done Sent false
85042024-02-20T22:45:08.299Z 14 Acked 1013 Read 6 Done Done Sent false
85052024-02-20T22:45:08.299Z 15 Acked 1014 Write 7 Done Done Sent false
85062024-02-20T22:45:08.299Z 16 Acked 1015 Read 10 Done Done Sent false
85072024-02-20T22:45:08.299Z 17 Acked 1016 Write 8 Done Done Sent false
85082024-02-20T22:45:08.299Z 18 Acked 1017 Write 5 Done Done Sent false
85092024-02-20T22:45:08.299Z 19 Acked 1018 Write 4 Done Done Sent false
85102024-02-20T22:45:08.299Z 20 Acked 1019 Write 9 Done Done Sent false
85112024-02-20T22:45:08.299Z 21 Acked 1020 Flush 0 Done Done Sent false
85122024-02-20T22:45:08.299Z 22 Acked 1021 Write 10 Done Done Sent false
85132024-02-20T22:45:08.299Z 23 Acked 1022 Write 4 Done Done Sent false
85142024-02-20T22:45:08.299Z 24 Acked 1023 Read 2 Done Done Sent false
85152024-02-20T22:45:08.299Z 25 Acked 1024 Write 9 Done Done Sent false
85162024-02-20T22:45:08.299Z 26 Acked 1025 Flush 0 Done Done Sent false
85172024-02-20T22:45:08.299Z 27 Acked 1026 Write 1 Done Done Sent false
85182024-02-20T22:45:08.299Z 28 Acked 1027 Read 1 Done Done Sent false
85192024-02-20T22:45:08.299Z 29 Acked 1028 Read 5 Done Done Sent false
85202024-02-20T22:45:08.300Z 30 Acked 1029 Read 8 Done Done Sent false
85212024-02-20T22:45:08.300Z STATES DS:0 DS:1 DS:2 TOTAL
85222024-02-20T22:45:08.300Z New 0 0 0 0
85232024-02-20T22:45:08.300Z Sent 0 0 28 28
85242024-02-20T22:45:08.300Z Done 30 30 2 62
85252024-02-20T22:45:08.300Z Skipped 0 0 0 0
85262024-02-20T22:45:08.300Z Error 0 0 0 0
85272024-02-20T22:45:08.300ZLast Flush: 1025 1025 0
85282024-02-20T22:45:08.300ZDownstairs last five completed:
85292024-02-20T22:45:08.300ZUpstairs last five completed: 30 29 28 27 26
85302024-02-20T22:45:08.305Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:07.824716998Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1450}
85312024-02-20T22:45:08.305ZWrote out file "/tmp/test_repair_verify.data"
85322024-02-20T22:45:08.305Z
85332024-02-20T22:45:08.306ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85342024-02-20T22:45:08.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85352024-02-20T22:45:08.320Z 0 000-019 74 74 74 133 133 133 F F F
85362024-02-20T22:45:08.320Z 1 020-039 74 74 74 133 133 133 F F F
85372024-02-20T22:45:08.320Z 2 040-059 77 77 77 136 136 136 F F F
85382024-02-20T22:45:08.320Z 3 060-079 77 77 77 136 136 136 F F F
85392024-02-20T22:45:08.320Z 4 080-099 75 75 75 134 134 134 F F F
85402024-02-20T22:45:08.320Z 5 100-119 65 65 65 115 115 115 F F F
85412024-02-20T22:45:08.320Z 6 120-139 74 74 74 133 133 133 F F F
85422024-02-20T22:45:08.320Z 7 140-159 75 75 75 134 134 134 F F F
85432024-02-20T22:45:08.320Z 8 160-179 78 78 75 138 138 134 F F F <---
85442024-02-20T22:45:08.320Z 9 180-199 78 78 74 138 138 133 F F F <---
85452024-02-20T22:45:08.320Z 10 200-219 78 78 72 138 138 128 F F F <---
85462024-02-20T22:45:08.328Z 11 220-239 74 74 74 133 133 133 F F F
85472024-02-20T22:45:08.328Z 12 240-259 77 77 77 136 136 136 F F F
85482024-02-20T22:45:08.328Z 13 260-279 73 73 73 130 130 130 F F F
85492024-02-20T22:45:08.328Z 14 280-299 77 77 77 136 136 136 F F F
85502024-02-20T22:45:08.328Z 15 300-319 77 77 77 136 136 136 F F F
85512024-02-20T22:45:08.328Z 16 320-339 78 78 76 138 138 135 F F F <---
85522024-02-20T22:45:08.329Z 17 340-359 78 78 71 138 138 126 T T F <---
85532024-02-20T22:45:08.329Z 18 360-379 75 75 75 134 134 134 F F F
85542024-02-20T22:45:08.329Z 19 380-399 78 78 71 137 137 126 F F F <---
85552024-02-20T22:45:08.329Z 20 400-419 78 78 71 137 137 126 F F F <---
85562024-02-20T22:45:08.329Z 21 420-439 71 71 71 126 126 126 F F F
85572024-02-20T22:45:08.329Z 22 440-459 76 76 76 135 135 135 F F F
85582024-02-20T22:45:08.329Z 23 460-479 78 78 76 137 137 135 F F T <---
85592024-02-20T22:45:08.329Z 24 480-499 78 78 73 137 137 130 F F T <---
85602024-02-20T22:45:08.329Z 25 500-519 78 78 71 137 137 126 F F F <---
85612024-02-20T22:45:08.329Z 26 520-539 78 78 76 137 137 135 F F F <---
85622024-02-20T22:45:08.329Z 27 540-559 77 77 77 136 136 136 F F F
85632024-02-20T22:45:08.329Z 28 560-579 78 78 77 137 137 136 F F F <---
85642024-02-20T22:45:08.329Z 29 580-599 78 78 72 137 137 128 F F F <---
85652024-02-20T22:45:08.329ZMax gen: 78, Max flush: 138
85662024-02-20T22:45:08.329ZError: Difference in extent metadata found!
85672024-02-20T22:45:08.329ZA Difference in extent metadata is expected here
85682024-02-20T22:45:08.329ZOn loop 39
85692024-02-20T22:45:08.329Z
85702024-02-20T22:45:08.329ZVerifying data now
85712024-02-20T22:45:12.938ZNo change after verify
85722024-02-20T22:45:12.943ZLoop: 39 Downstairs dump after verify (and repair):
85732024-02-20T22:45:12.954ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85742024-02-20T22:45:12.954Z 0 000-019 74 74 74 133 133 133 F F F
85752024-02-20T22:45:12.954Z 1 020-039 74 74 74 133 133 133 F F F
85762024-02-20T22:45:12.954Z 2 040-059 77 77 77 136 136 136 F F F
85772024-02-20T22:45:12.954Z 3 060-079 77 77 77 136 136 136 F F F
85782024-02-20T22:45:12.954Z 4 080-099 75 75 75 134 134 134 F F F
85792024-02-20T22:45:12.954Z 5 100-119 65 65 65 115 115 115 F F F
85802024-02-20T22:45:12.954Z 6 120-139 74 74 74 133 133 133 F F F
85812024-02-20T22:45:12.954Z 7 140-159 75 75 75 134 134 134 F F F
85822024-02-20T22:45:12.959Z 8 160-179 78 78 78 138 138 138 F F F
85832024-02-20T22:45:12.959Z 9 180-199 78 78 78 138 138 138 F F F
85842024-02-20T22:45:12.959Z 10 200-219 78 78 78 138 138 138 F F F
85852024-02-20T22:45:12.959Z 11 220-239 74 74 74 133 133 133 F F F
85862024-02-20T22:45:12.959Z 12 240-259 77 77 77 136 136 136 F F F
85872024-02-20T22:45:12.959Z 13 260-279 73 73 73 130 130 130 F F F
85882024-02-20T22:45:12.959Z 14 280-299 77 77 77 136 136 136 F F F
85892024-02-20T22:45:12.959Z 15 300-319 77 77 77 136 136 136 F F F
85902024-02-20T22:45:12.959Z 16 320-339 78 78 78 138 138 138 F F F
85912024-02-20T22:45:12.959Z 17 340-359 79 79 79 139 139 139 F F F
85922024-02-20T22:45:12.959Z 18 360-379 75 75 75 134 134 134 F F F
85932024-02-20T22:45:12.959Z 19 380-399 78 78 78 137 137 137 F F F
85942024-02-20T22:45:12.959Z 20 400-419 78 78 78 137 137 137 F F F
85952024-02-20T22:45:12.959Z 21 420-439 71 71 71 126 126 126 F F F
85962024-02-20T22:45:12.959Z 22 440-459 76 76 76 135 135 135 F F F
85972024-02-20T22:45:12.959Z 23 460-479 78 78 78 137 137 137 F F F
85982024-02-20T22:45:12.959Z 24 480-499 78 78 78 137 137 137 F F F
85992024-02-20T22:45:12.959Z 25 500-519 78 78 78 137 137 137 F F F
86002024-02-20T22:45:12.959Z 26 520-539 78 78 78 137 137 137 F F F
86012024-02-20T22:45:12.959Z 27 540-559 77 77 77 136 136 136 F F F
86022024-02-20T22:45:12.959Z 28 560-579 78 78 78 137 137 137 F F F
86032024-02-20T22:45:12.959Z 29 580-599 78 78 78 137 137 137 F F F
86042024-02-20T22:45:12.959ZMax gen: 79, Max flush: 139
86052024-02-20T22:45:12.960Z
86062024-02-20T22:45:12.960ZBegin loop 40
86072024-02-20T22:45:12.960ZDownstairs to restart: 1
86082024-02-20T22:45:12.975Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501046945Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458}
86092024-02-20T22:45:12.975Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501339065Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458}
86102024-02-20T22:45:12.975Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501373154Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458}
86112024-02-20T22:45:12.983Z{"msg":"Crucible f3f5515f-522c-47f0-b274-b1c98feb6339 has session id: 3c66e9a1-60ef-439a-9afc-9a231c97debc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501399563Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86122024-02-20T22:45:12.983Z{"msg":"Upstairs opts: Upstairs UUID: f3f5515f-522c-47f0-b274-b1c98feb6339, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501452861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86132024-02-20T22:45:12.983ZCrucible runtime is spawned
86142024-02-20T22:45:12.983Z{"msg":"Crucible stats registered with UUID: f3f5515f-522c-47f0-b274-b1c98feb6339","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501504549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86152024-02-20T22:45:12.983Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501548278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458}
86162024-02-20T22:45:12.983Z{"msg":"f3f5515f-522c-47f0-b274-b1c98feb6339 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501682123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86172024-02-20T22:45:12.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501709542Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86182024-02-20T22:45:12.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501724971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86192024-02-20T22:45:12.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50174687Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86202024-02-20T22:45:12.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50176172Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86212024-02-20T22:45:12.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501785339Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86222024-02-20T22:45:12.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501803338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86232024-02-20T22:45:12.983Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501824318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"io task","client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86242024-02-20T22:45:12.983Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501843847Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"io task","client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86252024-02-20T22:45:12.983Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.501862366Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"io task","client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86262024-02-20T22:45:12.983Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50202365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"io task","client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86272024-02-20T22:45:12.984Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502060149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"io task","client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86282024-02-20T22:45:12.984Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502086828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"io task","client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86292024-02-20T22:45:12.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502732185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86302024-02-20T22:45:12.984Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502774164Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86312024-02-20T22:45:12.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502800303Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86322024-02-20T22:45:12.984Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502828192Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86332024-02-20T22:45:12.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502855771Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86342024-02-20T22:45:12.984Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.502900229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86352024-02-20T22:45:12.984Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50343984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86362024-02-20T22:45:12.984Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.503478638Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86372024-02-20T22:45:12.984Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50371629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86382024-02-20T22:45:12.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50398616Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86392024-02-20T22:45:12.984Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504039308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86402024-02-20T22:45:12.984Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504076667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86412024-02-20T22:45:12.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504093336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86422024-02-20T22:45:12.984Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504115236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86432024-02-20T22:45:12.984Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504130055Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86442024-02-20T22:45:12.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504224772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86452024-02-20T22:45:12.984Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50426936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86462024-02-20T22:45:12.984Z{"msg":"[0]R flush_numbers[0..12]: [133, 133, 136, 136, 134, 115, 133, 134, 138, 138, 138, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504288809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86472024-02-20T22:45:12.984Z{"msg":"[0]R generation[0..12]: [74, 74, 77, 77, 75, 65, 74, 75, 78, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504311869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86482024-02-20T22:45:12.984Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504326808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86492024-02-20T22:45:12.984Z{"msg":"[1]R flush_numbers[0..12]: [133, 133, 136, 136, 134, 115, 133, 134, 138, 138, 138, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504342098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86502024-02-20T22:45:12.984Z{"msg":"[1]R generation[0..12]: [74, 74, 77, 77, 75, 65, 74, 75, 78, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504356057Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86512024-02-20T22:45:12.985Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504378366Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86522024-02-20T22:45:12.985Z{"msg":"[2]R flush_numbers[0..12]: [133, 133, 136, 136, 134, 115, 133, 134, 138, 138, 138, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504393016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86532024-02-20T22:45:12.985Z{"msg":"[2]R generation[0..12]: [74, 74, 77, 77, 75, 65, 74, 75, 78, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504415355Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458Wait for a query_work_queue command to finish before sending IO
86542024-02-20T22:45:12.985Z,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86552024-02-20T22:45:12.985Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504434364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86562024-02-20T22:45:12.985Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504447644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86572024-02-20T22:45:12.985Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504468193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86582024-02-20T22:45:12.985Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504481633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86592024-02-20T22:45:12.985Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504495472Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86602024-02-20T22:45:12.985Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504520311Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86612024-02-20T22:45:12.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.50454276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86622024-02-20T22:45:12.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504575739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86632024-02-20T22:45:12.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504600828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86642024-02-20T22:45:12.985Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504633767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86652024-02-20T22:45:12.985Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504657326Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86662024-02-20T22:45:12.985Z{"msg":"f3f5515f-522c-47f0-b274-b1c98feb6339 is now active with session: 3c66e9a1-60ef-439a-9afc-9a231c97debc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504671536Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86672024-02-20T22:45:12.985Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:12.504701315Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458,"session_id":"3c66e9a1-60ef-439a-9afc-9a231c97debc"}
86682024-02-20T22:45:12.985Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86692024-02-20T22:45:12.985Z,"time":"2024-02-20T22:45:12.504720414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458}
86702024-02-20T22:45:12.985ZLoading write count information from file "/tmp/test_repair_verify.data"
86712024-02-20T22:45:12.985ZRun Repair workload
86722024-02-20T22:45:12.985Z01/30 Write block 476 len 40960 data: 9 13 13 10 12 12 11 11 9 7
86732024-02-20T22:45:12.991Z02/30 Flush
86742024-02-20T22:45:12.991Z03/30 Write block 561 len 32768 data: 6 4 5 7 6 7 7 7
86752024-02-20T22:45:12.991Z04/30 Write block 320 len 36864 data: 11 9 11 7 3 6 7 7 7
86762024-02-20T22:45:12.991Z05/30 Flush
86772024-02-20T22:45:12.991Z06/30 Write block 512 len 28672 data: 7 7 7 8 6 5 3
86782024-02-20T22:45:12.991Z07/30 Write block 336 len 28672 data: 8 8 7 8 7 8 8
86792024-02-20T22:45:12.991Z08/30 Read block 257 len 12288
86802024-02-20T22:45:12.991Z09/30 Write block 302 len 32768 data: 12 12 12 11 9 9 8 10
86812024-02-20T22:45:12.994Z10/30 Write block 229 len 40960 data: 10 12 11 12 14 13 9 9 9 6
86822024-02-20T22:45:12.995Z11/30 Write block 258 len 20480 data: 5 5 4 4 4
86832024-02-20T22:45:12.995Z12/30 Write block 447 len 24576 data: 8 7 6 6 6 9
86842024-02-20T22:45:12.995Z13/30 Read block 175 len 36864
86852024-02-20T22:45:12.998Z14/30 Write block 589 len 4096 data: 8
86862024-02-20T22:45:12.998Z15/30 Write block 113 len 28672 data: 10 8 7 8 9 5 7
86872024-02-20T22:45:12.998Z16/30 Read block 7 len 12288
86882024-02-20T22:45:12.999Z17/30 Write block 577 len 16384 data: 7 8 10 10
86892024-02-20T22:45:12.999Z18/30 Read block 70 len 4096
86902024-02-20T22:45:12.999Z19/30 Read block 65 len 12288
86912024-02-20T22:45:13.003Z20/30 Write block 155 len 12288 data: 6 7 8
86922024-02-20T22:45:13.003Z21/30 Read block 589 len 12288
86932024-02-20T22:45:13.003Z22/30 Write block 174 len 20480 data: 7 9 8 8 8
86942024-02-20T22:45:13.003Z23/30 Read block 540 len 40960
86952024-02-20T22:45:13.003Z24/30 Read block 18 len 4096
86962024-02-20T22:45:13.008Z25/30 Write block 105 len 32768 data: 4 4 4 5 6 8 9 9
86972024-02-20T22:45:13.008Z26/30 Write block 91 len 20480 data: 6 3 3 3 3
86982024-02-20T22:45:13.008Z27/30 Write block 546 len 24576 data: 6 8 8 8 6 6
86992024-02-20T22:45:13.008Z28/30 Write block 97 len 4096 data: 3
87002024-02-20T22:45:13.008Z29/30 Write block 36 len 32768 data: 9 8 9 10 8 6 7 7
87012024-02-20T22:45:13.008Z30/30 Write block 351 len 16384 data: 5 4 4 4
87022024-02-20T22:45:13.008Z----------------------------------------------------------------
87032024-02-20T22:45:13.008Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:28
87042024-02-20T22:45:13.008ZStates: Active Active Active
87052024-02-20T22:45:13.008ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87062024-02-20T22:45:13.008Z 3 Acked 1002 Write 8 Done Sent Done false
87072024-02-20T22:45:13.008Z 4 Acked 1003 Write 9 Done Sent Done false
87082024-02-20T22:45:13.008Z 5 Acked 1004 Flush 0 Done Sent Done false
87092024-02-20T22:45:13.008Z 6 Acked 1005 Write 7 Done Sent Done false
87102024-02-20T22:45:13.008Z 7 Acked 1006 Write 7 Done Sent Done false
87112024-02-20T22:45:13.008Z 8 Acked 1007 Read 3 Done Sent Done false
87122024-02-20T22:45:13.008Z 9 Acked 1008 Write 8 Done Sent Done false
87132024-02-20T22:45:13.008Z 10 Acked 1009 Write 10 Done Sent Done false
87142024-02-20T22:45:13.008Z 11 Acked 1010 Write 5 Done Sent Done false
87152024-02-20T22:45:13.008Z 12 Acked 1011 Write 6 Done Sent Done false
87162024-02-20T22:45:13.008Z 13 Acked 1012 Read 9 Done Sent Done false
87172024-02-20T22:45:13.008Z 14 Acked 1013 Write 1 Done Sent Done false
87182024-02-20T22:45:13.009Z 15 Acked 1014 Write 7 Done Sent Done false
87192024-02-20T22:45:13.009Z 16 Acked 1015 Read 3 Done Sent Done false
87202024-02-20T22:45:13.009Z 17 Acked 1016 Write 4 Done Sent Done false
87212024-02-20T22:45:13.009Z 18 Acked 1017 Read 1 Done Sent Done false
87222024-02-20T22:45:13.009Z 19 Acked 1018 Read 3 Done Sent Done false
87232024-02-20T22:45:13.009Z 20 Acked 1019 Write 3 Done Sent Done false
87242024-02-20T22:45:13.009Z 21 Acked 1020 Read 3 Done Sent Done false
87252024-02-20T22:45:13.009Z 22 Acked 1021 Write 5 Done Sent Done false
87262024-02-20T22:45:13.009Z 23 Acked 1022 Read 10 Done Sent Done false
87272024-02-20T22:45:13.009Z 24 Acked 1023 Read 1 Done Sent Done false
87282024-02-20T22:45:13.013Z 25 Acked 1024 Write 8 Done Sent Done false
87292024-02-20T22:45:13.013Z 26 Acked 1025 Write 5 Done Sent Done false
87302024-02-20T22:45:13.013Z 27 Acked 1026 Write 6 Done Sent Done false
87312024-02-20T22:45:13.013Z 28 Acked 1027 Write 1 Done Sent Done false
87322024-02-20T22:45:13.013Z 29 Acked 1028 Write 8 Done Sent Sent false
87332024-02-20T22:45:13.013Z 30 Acked 1029 Write 4 Sent Sent Sent false
87342024-02-20T22:45:13.013Z STATES DS:0 DS:1 DS:2 TOTAL
87352024-02-20T22:45:13.013Z New 0 0 0 0
87362024-02-20T22:45:13.013Z Sent 1 28 2 31
87372024-02-20T22:45:13.013Z Done 27 0 26 53
87382024-02-20T22:45:13.013Z Skipped 0 0 0 0
87392024-02-20T22:45:13.013Z Error 0 0 0 0
87402024-02-20T22:45:13.013ZLast Flush: 1004 1001 1004
87412024-02-20T22:45:13.014ZDownstairs last five completed: 1001 1000
87422024-02-20T22:45:13.014ZUpstairs last five completed: 30 29 28 27 26
87432024-02-20T22:45:13.014Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:12.534012296Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1458}
87442024-02-20T22:45:13.014ZWrote out file "/tmp/test_repair_verify.data"
87452024-02-20T22:45:13.014Z
87462024-02-20T22:45:13.017ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87472024-02-20T22:45:13.035ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87482024-02-20T22:45:13.035Z 0 000-019 74 74 74 133 133 133 F F F
87492024-02-20T22:45:13.035Z 1 020-039 74 74 74 133 133 133 T F T <---
87502024-02-20T22:45:13.035Z 2 040-059 77 77 77 136 136 136 T F T <---
87512024-02-20T22:45:13.035Z 3 060-079 77 77 77 136 136 136 F F F
87522024-02-20T22:45:13.035Z 4 080-099 75 75 75 134 134 134 T F T <---
87532024-02-20T22:45:13.035Z 5 100-119 65 65 65 115 115 115 T F T <---
87542024-02-20T22:45:13.035Z 6 120-139 74 74 74 133 133 133 F F F
87552024-02-20T22:45:13.035Z 7 140-159 75 75 75 134 134 134 T F T <---
87562024-02-20T22:45:13.036Z 8 160-179 78 78 78 138 138 138 T F T <---
87572024-02-20T22:45:13.036Z 9 180-199 78 78 78 138 138 138 F F F
87582024-02-20T22:45:13.036Z 10 200-219 78 78 78 138 138 138 F F F
87592024-02-20T22:45:13.036Z 11 220-239 74 74 74 133 133 133 T F T <---
87602024-02-20T22:45:13.042Z 12 240-259 77 77 77 136 136 136 T F T <---
87612024-02-20T22:45:13.042Z 13 260-279 73 73 73 130 130 130 T F T <---
87622024-02-20T22:45:13.042Z 14 280-299 77 77 77 136 136 136 F F F
87632024-02-20T22:45:13.042Z 15 300-319 77 77 77 136 136 136 T F T <---
87642024-02-20T22:45:13.043Z 16 320-339 80 78 80 141 138 141 T F T <---
87652024-02-20T22:45:13.043Z 17 340-359 79 79 79 139 139 139 T F T <---
87662024-02-20T22:45:13.043Z 18 360-379 75 75 75 134 134 134 F F F
87672024-02-20T22:45:13.043Z 19 380-399 78 78 78 137 137 137 F F F
87682024-02-20T22:45:13.043Z 20 400-419 78 78 78 137 137 137 F F F
87692024-02-20T22:45:13.043Z 21 420-439 71 71 71 126 126 126 F F F
87702024-02-20T22:45:13.043Z 22 440-459 76 76 76 135 135 135 T F T <---
87712024-02-20T22:45:13.043Z 23 460-479 80 80 80 140 140 140 F F F
87722024-02-20T22:45:13.043Z 24 480-499 80 80 80 140 140 140 F F F
87732024-02-20T22:45:13.043Z 25 500-519 78 78 78 137 137 137 T F T <---
87742024-02-20T22:45:13.043Z 26 520-539 78 78 78 137 137 137 F F F
87752024-02-20T22:45:13.043Z 27 540-559 77 77 77 136 136 136 T F T <---
87762024-02-20T22:45:13.043Z 28 560-579 80 78 80 141 137 141 T F T <---
87772024-02-20T22:45:13.043Z 29 580-599 78 78 78 137 137 137 T F T <---
87782024-02-20T22:45:13.043ZMax gen: 80, Max flush: 141
87792024-02-20T22:45:13.043ZError: Difference in extent metadata found!
87802024-02-20T22:45:13.043ZA Difference in extent metadata is expected here
87812024-02-20T22:45:13.043ZOn loop 40
87822024-02-20T22:45:13.043Z
87832024-02-20T22:45:13.047ZVerifying data now
87842024-02-20T22:45:17.833ZNo change after verify
87852024-02-20T22:45:17.838ZLoop: 40 Downstairs dump after verify (and repair):
87862024-02-20T22:45:17.847ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87872024-02-20T22:45:17.847Z 0 000-019 74 74 74 133 133 133 F F F
87882024-02-20T22:45:17.847Z 1 020-039 81 81 81 142 142 142 F F F
87892024-02-20T22:45:17.847Z 2 040-059 81 81 81 142 142 142 F F F
87902024-02-20T22:45:17.847Z 3 060-079 77 77 77 136 136 136 F F F
87912024-02-20T22:45:17.847Z 4 080-099 81 81 81 142 142 142 F F F
87922024-02-20T22:45:17.847Z 5 100-119 81 81 81 142 142 142 F F F
87932024-02-20T22:45:17.847Z 6 120-139 74 74 74 133 133 133 F F F
87942024-02-20T22:45:17.847Z 7 140-159 81 81 81 142 142 142 F F F
87952024-02-20T22:45:17.847Z 8 160-179 81 81 81 142 142 142 F F F
87962024-02-20T22:45:17.847Z 9 180-199 78 78 78 138 138 138 F F F
87972024-02-20T22:45:17.853Z 10 200-219 78 78 78 138 138 138 F F F
87982024-02-20T22:45:17.853Z 11 220-239 81 81 81 142 142 142 F F F
87992024-02-20T22:45:17.853Z 12 240-259 81 81 81 142 142 142 F F F
88002024-02-20T22:45:17.853Z 13 260-279 81 81 81 142 142 142 F F F
88012024-02-20T22:45:17.853Z 14 280-299 77 77 77 136 136 136 F F F
88022024-02-20T22:45:17.853Z 15 300-319 81 81 81 142 142 142 F F F
88032024-02-20T22:45:17.853Z 16 320-339 81 81 81 142 142 142 F F F
88042024-02-20T22:45:17.854Z 17 340-359 81 81 81 142 142 142 F F F
88052024-02-20T22:45:17.854Z 18 360-379 75 75 75 134 134 134 F F F
88062024-02-20T22:45:17.854Z 19 380-399 78 78 78 137 137 137 F F F
88072024-02-20T22:45:17.854Z 20 400-419 78 78 78 137 137 137 F F F
88082024-02-20T22:45:17.854Z 21 420-439 71 71 71 126 126 126 F F F
88092024-02-20T22:45:17.854Z 22 440-459 81 81 81 142 142 142 F F F
88102024-02-20T22:45:17.854Z 23 460-479 80 80 80 140 140 140 F F F
88112024-02-20T22:45:17.854Z 24 480-499 80 80 80 140 140 140 F F F
88122024-02-20T22:45:17.854Z 25 500-519 81 81 81 142 142 142 F F F
88132024-02-20T22:45:17.854Z 26 520-539 78 78 78 137 137 137 F F F
88142024-02-20T22:45:17.854Z 27 540-559 81 81 81 142 142 142 F F F
88152024-02-20T22:45:17.854Z 28 560-579 81 81 81 142 142 142 F F F
88162024-02-20T22:45:17.854Z 29 580-599 81 81 81 142 142 142 F F F
88172024-02-20T22:45:17.854ZMax gen: 81, Max flush: 142
88182024-02-20T22:45:17.854Z
88192024-02-20T22:45:17.855ZBegin loop 41
88202024-02-20T22:45:17.855ZDownstairs to restart: 0
88212024-02-20T22:45:17.870Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.393875195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466}
88222024-02-20T22:45:17.870Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\nCrucible runtime is spawned
88232024-02-20T22:45:17.878Z rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.39428699Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466}
88242024-02-20T22:45:17.878Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394346179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466}
88252024-02-20T22:45:17.878Z{"msg":"Crucible 561edd2b-d02b-4953-8211-8ef33d4ca1c9 has session id: da6f49aa-1ae4-4917-9361-a6b6cc2d4e39","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394376557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88262024-02-20T22:45:17.878Z{"msg":"Upstairs opts: Upstairs UUID: 561edd2b-d02b-4953-8211-8ef33d4ca1c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394420166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88272024-02-20T22:45:17.879Z{"msg":"Crucible stats registered with UUID: 561edd2b-d02b-4953-8211-8ef33d4ca1c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394437805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88282024-02-20T22:45:17.879Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394459024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466}
88292024-02-20T22:45:17.879Z{"msg":"561edd2b-d02b-4953-8211-8ef33d4ca1c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394532942Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88302024-02-20T22:45:17.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394564701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88312024-02-20T22:45:17.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394603329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88322024-02-20T22:45:17.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394631768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88332024-02-20T22:45:17.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394658237Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88342024-02-20T22:45:17.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394682846Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88352024-02-20T22:45:17.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394723495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88362024-02-20T22:45:17.879Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394744324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"io task","client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88372024-02-20T22:45:17.879Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394768583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"io task","client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88382024-02-20T22:45:17.879Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394802252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88392024-02-20T22:45:17.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394883269Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"io task","client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88402024-02-20T22:45:17.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394945007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88412024-02-20T22:45:17.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.394978066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"io task","client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88422024-02-20T22:45:17.879Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.395576964Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88432024-02-20T22:45:17.879Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.395624413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88442024-02-20T22:45:17.879Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.395655412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88452024-02-20T22:45:17.879Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.395683121Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88462024-02-20T22:45:17.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.39570725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88472024-02-20T22:45:17.880Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.395752918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88482024-02-20T22:45:17.880Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396241241Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88492024-02-20T22:45:17.880Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396376436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88502024-02-20T22:45:17.880Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396697044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88512024-02-20T22:45:17.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396746793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88522024-02-20T22:45:17.880Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396774741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88532024-02-20T22:45:17.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396804751Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88542024-02-20T22:45:17.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396919306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88552024-02-20T22:45:17.880Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396961395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88562024-02-20T22:45:17.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.396977264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88572024-02-20T22:45:17.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397246155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88582024-02-20T22:45:17.880Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397287943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88592024-02-20T22:45:17.880Z{"msg":"[0]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 142, 142, 138, 138, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397349131Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88602024-02-20T22:45:17.880Z{"msg":"[0]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 81, 81, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.39737673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88612024-02-20T22:45:17.880Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397395999Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88622024-02-20T22:45:17.880Z{"msg":"[1]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 142, 142, 138, 138, 142]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
88632024-02-20T22:45:17.880Z,"time":"2024-02-20T22:45:17.397432228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88642024-02-20T22:45:17.880Z{"msg":"[1]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 81, 81, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397450977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88652024-02-20T22:45:17.880Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397482686Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88662024-02-20T22:45:17.880Z{"msg":"[2]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 142, 142, 138, 138, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397502635Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88672024-02-20T22:45:17.881Z{"msg":"[2]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 81, 81, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397532894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88682024-02-20T22:45:17.881Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397553284Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88692024-02-20T22:45:17.881Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397581743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88702024-02-20T22:45:17.881Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397599812Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88712024-02-20T22:45:17.881Z{"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397628071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88722024-02-20T22:45:17.881Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.39764764Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88732024-02-20T22:45:17.881Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397676659Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88742024-02-20T22:45:17.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397690669Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88752024-02-20T22:45:17.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397721918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88762024-02-20T22:45:17.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397752077Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88772024-02-20T22:45:17.881Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397785245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88782024-02-20T22:45:17.881Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397813794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88792024-02-20T22:45:17.881Z{"msg":"561edd2b-d02b-4953-8211-8ef33d4ca1c9 is now active with session: da6f49aa-1ae4-4917-9361-a6b6cc2d4e39","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397846783Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88802024-02-20T22:45:17.881Z:1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88812024-02-20T22:45:17.881Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397879802Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466,"session_id":"da6f49aa-1ae4-4917-9361-a6b6cc2d4e39"}
88822024-02-20T22:45:17.881Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:17.397911141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466}
88832024-02-20T22:45:17.881ZLoading write count information from file "/tmp/test_repair_verify.data"
88842024-02-20T22:45:17.881ZRun Repair workload
88852024-02-20T22:45:17.881Z01/30 Write block 219 len 24576 data: 4 5 5 5 5 4
88862024-02-20T22:45:17.881Z02/30 Read block 445 len 40960
88872024-02-20T22:45:17.892Z03/30 Read block 40 len 4096
88882024-02-20T22:45:17.892Z04/30 Read block 508 len 20480
88892024-02-20T22:45:17.892Z05/30 Read block 431 len 8192
88902024-02-20T22:45:17.893Z06/30 Read block 22 len 20480
88912024-02-20T22:45:17.893Z07/30 Read block 397 len 8192
88922024-02-20T22:45:17.893Z08/30 Read block 133 len 32768
88932024-02-20T22:45:17.893Z09/30 Read block 308 len 28672
88942024-02-20T22:45:17.893Z10/30 Read block 338 len 16384
88952024-02-20T22:45:17.893Z11/30 Read block 502 len 24576
88962024-02-20T22:45:17.893Z12/30 Read block 394 len 16384
88972024-02-20T22:45:17.893Z13/30 Read block 69 len 24576
88982024-02-20T22:45:17.893Z14/30 Read block 321 len 24576
88992024-02-20T22:45:17.893Z15/30 Read block 40 len 36864
89002024-02-20T22:45:17.900Z16/30 Write block 151 len 4096 data: 11
89012024-02-20T22:45:17.900Z17/30 Read block 54 len 32768
89022024-02-20T22:45:17.900Z18/30 Read block 286 len 8192
89032024-02-20T22:45:17.900Z19/30 Flush
89042024-02-20T22:45:17.900Z20/30 Write block 587 len 24576 data: 6 7 9 10 10 9
89052024-02-20T22:45:17.900Z21/30 Read block 580 len 4096
89062024-02-20T22:45:17.900Z22/30 Read block 23 len 12288
89072024-02-20T22:45:17.900Z23/30 Write block 574 len 32768 data: 3 4 6 8 9 11 11 9
89082024-02-20T22:45:17.900Z24/30 Read block 505 len 40960
89092024-02-20T22:45:17.900Z25/30 Flush
89102024-02-20T22:45:17.900Z26/30 Write block 520 len 12288 data: 3 4 5
89112024-02-20T22:45:17.900Z27/30 Read block 284 len 40960
89122024-02-20T22:45:17.900Z28/30 Write block 590 len 28672 data: 11 11 10 9 10 10 9
89132024-02-20T22:45:17.900Z29/30 Write block 413 len 32768 data: 10 10 9 7 5 6 8 8
89142024-02-20T22:45:17.910Z30/30 Read block 184 len 12288
89152024-02-20T22:45:17.910Z----------------------------------------------------------------
89162024-02-20T22:45:17.910Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89172024-02-20T22:45:17.910ZStates: Active Active Active
89182024-02-20T22:45:17.910ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89192024-02-20T22:45:17.911Z 1 Acked 1000 Write 6 Done Done Done false
89202024-02-20T22:45:17.911Z 2 Acked 1001 Read 10 Done Done Done false
89212024-02-20T22:45:17.911Z 3 Acked 1002 Read 1 Done Done Done false
89222024-02-20T22:45:17.911Z 4 Acked 1003 Read 5 Done Done Done false
89232024-02-20T22:45:17.911Z 5 Acked 1004 Read 2 Done Done Done false
89242024-02-20T22:45:17.911Z 6 Acked 1005 Read 5 Done Done Done false
89252024-02-20T22:45:17.911Z 7 Acked 1006 Read 2 Done Done Done false
89262024-02-20T22:45:17.911Z 8 Acked 1007 Read 8 Done Done Done false
89272024-02-20T22:45:17.911Z 9 Acked 1008 Read 7 Sent Done Done false
89282024-02-20T22:45:17.911Z 10 Acked 1009 Read 4 Sent Done Done false
89292024-02-20T22:45:17.911Z 11 Acked 1010 Read 6 Sent Done Done false
89302024-02-20T22:45:17.911Z 12 Acked 1011 Read 4 Sent Done Done false
89312024-02-20T22:45:17.911Z 13 Acked 1012 Read 6 Sent Done Done false
89322024-02-20T22:45:17.911Z 14 Acked 1013 Read 6 Sent Done Done false
89332024-02-20T22:45:17.911Z 15 Acked 1014 Read 9 Sent Done Done false
89342024-02-20T22:45:17.911Z 16 Acked 1015 Write 1 Sent Done Done false
89352024-02-20T22:45:17.911Z 17 Acked 1016 Read 8 Sent Done Done false
89362024-02-20T22:45:17.911Z 18 Acked 1017 Read 2 Sent Done Done false
89372024-02-20T22:45:17.911Z 19 Acked 1018 Flush 0 Sent Done Done false
89382024-02-20T22:45:17.911Z 20 Acked 1019 Write 6 Sent Done Done false
89392024-02-20T22:45:17.911Z 21 Acked 1020 Read 1 Sent Done Done false
89402024-02-20T22:45:17.911Z 22 Acked 1021 Read 3 Sent Done Done false
89412024-02-20T22:45:17.911Z 23 Acked 1022 Write 8 Sent Done Done false
89422024-02-20T22:45:17.912Z 24 Acked 1023 Read 10 Sent Done Done false
89432024-02-20T22:45:17.912Z 25 Acked 1024 Flush 0 Sent Done Done false
89442024-02-20T22:45:17.912Z 26 Acked 1025 Write 3 Sent Done Done false
89452024-02-20T22:45:17.912Z 27 Acked 1026 Read 10 Sent Done Done false
89462024-02-20T22:45:17.912Z 28 Acked 1027 Write 7 Sent Done Done false
89472024-02-20T22:45:17.912Z 29 Acked 1028 Write 8 Sent Done Done false
89482024-02-20T22:45:17.912Z 30 Acked 1029 Read 3 Sent Done Sent false
89492024-02-20T22:45:17.912Z STATES DS:0 DS:1 DS:2 TOTAL
89502024-02-20T22:45:17.912Z New 0 0 0 0
89512024-02-20T22:45:17.912Z Sent 22 0 1 23
89522024-02-20T22:45:17.912Z Done 8 30 29 67
89532024-02-20T22:45:17.912Z Skipped 0 0 0 0
89542024-02-20T22:45:17.912Z Error 0 0 0 0
89552024-02-20T22:45:17.912ZLast Flush: 0 1024 1024
89562024-02-20T22:45:17.912ZDownstairs last five completed:
89572024-02-20T22:45:17.912ZUpstairs last five completed: 30 29 28 27 26
89582024-02-20T22:45:17.912Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:17.427468643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1466}
89592024-02-20T22:45:17.912ZWrote out file "/tmp/test_repair_verify.data"
89602024-02-20T22:45:17.917Z
89612024-02-20T22:45:17.917ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89622024-02-20T22:45:17.924ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89632024-02-20T22:45:17.925Z 0 000-019 74 74 74 133 133 133 F F F
89642024-02-20T22:45:17.925Z 1 020-039 81 81 81 142 142 142 F F F
89652024-02-20T22:45:17.925Z 2 040-059 81 81 81 142 142 142 F F F
89662024-02-20T22:45:17.925Z 3 060-079 77 77 77 136 136 136 F F F
89672024-02-20T22:45:17.925Z 4 080-099 81 81 81 142 142 142 F F F
89682024-02-20T22:45:17.925Z 5 100-119 81 81 81 142 142 142 F F F
89692024-02-20T22:45:17.925Z 6 120-139 74 74 74 133 133 133 F F F
89702024-02-20T22:45:17.925Z 7 140-159 81 82 82 142 143 143 F F F <---
89712024-02-20T22:45:17.925Z 8 160-179 81 81 81 142 142 142 F F F
89722024-02-20T22:45:17.925Z 9 180-199 78 78 78 138 138 138 F F F
89732024-02-20T22:45:17.925Z 10 200-219 78 82 82 138 143 143 T F F <---
89742024-02-20T22:45:17.925Z 11 220-239 81 82 82 142 143 143 T F F <---
89752024-02-20T22:45:17.925Z 12 240-259 81 81 81 142 142 142 F F F
89762024-02-20T22:45:17.925Z 13 260-279 81 81 81 142 142 142 F F F
89772024-02-20T22:45:17.925Z 14 280-299 77 77 77 136 136 136 F F F
89782024-02-20T22:45:17.925Z 15 300-319 81 81 81 142 142 142 F F F
89792024-02-20T22:45:17.935Z 16 320-339 81 81 81 142 142 142 F F F
89802024-02-20T22:45:17.935Z 17 340-359 81 81 81 142 142 142 F F F
89812024-02-20T22:45:17.935Z 18 360-379 75 75 75 134 134 134 F F F
89822024-02-20T22:45:17.935Z 19 380-399 78 78 78 137 137 137 F F F
89832024-02-20T22:45:17.935Z 20 400-419 78 78 78 137 137 137 F T T <---
89842024-02-20T22:45:17.935Z 21 420-439 71 71 71 126 126 126 F T T <---
89852024-02-20T22:45:17.935Z 22 440-459 81 81 81 142 142 142 F F F
89862024-02-20T22:45:17.935Z 23 460-479 80 80 80 140 140 140 F F F
89872024-02-20T22:45:17.935Z 24 480-499 80 80 80 140 140 140 F F F
89882024-02-20T22:45:17.935Z 25 500-519 81 81 81 142 142 142 F F F
89892024-02-20T22:45:17.935Z 26 520-539 78 78 78 137 137 137 F T T <---
89902024-02-20T22:45:17.935Z 27 540-559 81 81 81 142 142 142 F F F
89912024-02-20T22:45:17.935Z 28 560-579 81 82 82 142 144 144 F F F <---
89922024-02-20T22:45:17.935Z 29 580-599 81 82 82 142 144 144 F T T <---
89932024-02-20T22:45:17.935ZMax gen: 82, Max flush: 144
89942024-02-20T22:45:17.935ZError: Difference in extent metadata found!
89952024-02-20T22:45:17.935ZA Difference in extent metadata is expected here
89962024-02-20T22:45:17.935ZOn loop 41
89972024-02-20T22:45:17.935Z
89982024-02-20T22:45:17.935ZVerifying data now
89992024-02-20T22:45:22.356ZNo change after verify
90002024-02-20T22:45:22.356ZLoop: 41 Downstairs dump after verify (and repair):
90012024-02-20T22:45:22.369ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90022024-02-20T22:45:22.369Z 0 000-019 74 74 74 133 133 133 F F F
90032024-02-20T22:45:22.369Z 1 020-039 81 81 81 142 142 142 F F F
90042024-02-20T22:45:22.370Z 2 040-059 81 81 81 142 142 142 F F F
90052024-02-20T22:45:22.370Z 3 060-079 77 77 77 136 136 136 F F F
90062024-02-20T22:45:22.370Z 4 080-099 81 81 81 142 142 142 F F F
90072024-02-20T22:45:22.370Z 5 100-119 81 81 81 142 142 142 F F F
90082024-02-20T22:45:22.370Z 6 120-139 74 74 74 133 133 133 F F F
90092024-02-20T22:45:22.370Z 7 140-159 82 82 82 143 143 143 F F F
90102024-02-20T22:45:22.370Z 8 160-179 81 81 81 142 142 142 F F F
90112024-02-20T22:45:22.370Z 9 180-199 78 78 78 138 138 138 F F F
90122024-02-20T22:45:22.370Z 10 200-219 82 82 82 143 143 143 F F F
90132024-02-20T22:45:22.370Z 11 220-239 82 82 82 143 143 143 F F F
90142024-02-20T22:45:22.370Z 12 240-259 81 81 81 142 142 142 F F F
90152024-02-20T22:45:22.374Z 13 260-279 81 81 81 142 142 142 F F F
90162024-02-20T22:45:22.374Z 14 280-299 77 77 77 136 136 136 F F F
90172024-02-20T22:45:22.375Z 15 300-319 81 81 81 142 142 142 F F F
90182024-02-20T22:45:22.375Z 16 320-339 81 81 81 142 142 142 F F F
90192024-02-20T22:45:22.375Z 17 340-359 81 81 81 142 142 142 F F F
90202024-02-20T22:45:22.375Z 18 360-379 75 75 75 134 134 134 F F F
90212024-02-20T22:45:22.375Z 19 380-399 78 78 78 137 137 137 F F F
90222024-02-20T22:45:22.375Z 20 400-419 83 83 83 145 145 145 F F F
90232024-02-20T22:45:22.375Z 21 420-439 83 83 83 145 145 145 F F F
90242024-02-20T22:45:22.375Z 22 440-459 81 81 81 142 142 142 F F F
90252024-02-20T22:45:22.375Z 23 460-479 80 80 80 140 140 140 F F F
90262024-02-20T22:45:22.375Z 24 480-499 80 80 80 140 140 140 F F F
90272024-02-20T22:45:22.375Z 25 500-519 81 81 81 142 142 142 F F F
90282024-02-20T22:45:22.375Z 26 520-539 83 83 83 145 145 145 F F F
90292024-02-20T22:45:22.375Z 27 540-559 81 81 81 142 142 142 F F F
90302024-02-20T22:45:22.375Z 28 560-579 82 82 82 144 144 144 F F F
90312024-02-20T22:45:22.375Z 29 580-599 83 83 83 145 145 145 F F F
90322024-02-20T22:45:22.375ZMax gen: 83, Max flush: 145
90332024-02-20T22:45:22.375Z
90342024-02-20T22:45:22.375ZBegin loop 42
90352024-02-20T22:45:22.375ZDownstairs to restart: 1
90362024-02-20T22:45:22.392Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917001155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474}
90372024-02-20T22:45:22.392Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917261716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474}
90382024-02-20T22:45:22.392Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917286235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474}
90392024-02-20T22:45:22.402Z{"msg":"Crucible fc7c1b62-6f2e-4666-9318-aa6e0492a8e8 has session id: 09cb6a6e-da97-442e-9964-f64026a3974a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917320723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90402024-02-20T22:45:22.402Z{"msg":"Crucible runtime is spawned
90412024-02-20T22:45:22.402ZUpstairs opts: Upstairs UUID: fc7c1b62-6f2e-4666-9318-aa6e0492a8e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917371222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90422024-02-20T22:45:22.402Z{"msg":"Crucible stats registered with UUID: fc7c1b62-6f2e-4666-9318-aa6e0492a8e8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.91740952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90432024-02-20T22:45:22.402Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917438969Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474}
90442024-02-20T22:45:22.402Z{"msg":"fc7c1b62-6f2e-4666-9318-aa6e0492a8e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917567545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90452024-02-20T22:45:22.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917655172Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90462024-02-20T22:45:22.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.91770356Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90472024-02-20T22:45:22.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917721529Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90482024-02-20T22:45:22.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917740248Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90492024-02-20T22:45:22.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917764968Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90502024-02-20T22:45:22.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.917860044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90512024-02-20T22:45:22.402Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918010119Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90522024-02-20T22:45:22.403Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918084876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90532024-02-20T22:45:22.403Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918121805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90542024-02-20T22:45:22.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918163204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90552024-02-20T22:45:22.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918192222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90562024-02-20T22:45:22.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918232061Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90572024-02-20T22:45:22.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.918993434Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90582024-02-20T22:45:22.403Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.919029633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90592024-02-20T22:45:22.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.919047092Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90602024-02-20T22:45:22.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.919069071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90612024-02-20T22:45:22.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.91908322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90622024-02-20T22:45:22.403Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.91910019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90632024-02-20T22:45:22.403Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.919635281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90642024-02-20T22:45:22.403Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.919790465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90652024-02-20T22:45:22.403Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.919834254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90662024-02-20T22:45:22.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920067175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90672024-02-20T22:45:22.403Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920110934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90682024-02-20T22:45:22.403Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920127873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90692024-02-20T22:45:22.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920385154Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90702024-02-20T22:45:22.403Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920452251Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90712024-02-20T22:45:22.403Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.92049054Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90722024-02-20T22:45:22.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.92050614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90732024-02-20T22:45:22.403Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920527829Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90742024-02-20T22:45:22.403Z{"msg":"[0]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 143, 142, 138, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920542578Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90752024-02-20T22:45:22.403Z{"msg":"[0]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 82, 81, 78, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920563267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90762024-02-20T22:45:22.404Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920585227Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90772024-02-20T22:45:22.404ZWait for a query_work_queue command to finish before sending IO
90782024-02-20T22:45:22.404Z{"msg":"[1]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 143, 142, 138, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920627925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90792024-02-20T22:45:22.404Z{"msg":"[1]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 82, 81, 78, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920661124Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90802024-02-20T22:45:22.404Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920677023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90812024-02-20T22:45:22.404Z{"msg":"[2]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 143, 142, 138, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920706362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90822024-02-20T22:45:22.404Z{"msg":"[2]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 82, 81, 78, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920744871Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90832024-02-20T22:45:22.404Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.92077346Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90842024-02-20T22:45:22.404Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.92078755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90852024-02-20T22:45:22.404Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920820538Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90862024-02-20T22:45:22.404Z{"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920856147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90872024-02-20T22:45:22.404Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920881936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90882024-02-20T22:45:22.404Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920896596Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90892024-02-20T22:45:22.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920928125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90902024-02-20T22:45:22.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920962103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90912024-02-20T22:45:22.404Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.920988162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90922024-02-20T22:45:22.404Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.921002342Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90932024-02-20T22:45:22.404Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.921034311Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90942024-02-20T22:45:22.404Z{"msg":"fc7c1b62-6f2e-4666-9318-aa6e0492a8e8 is now active with session: 09cb6a6e-da97-442e-9964-f64026a3974a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.921066289Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90952024-02-20T22:45:22.404Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.921080509Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474,"session_id":"09cb6a6e-da97-442e-9964-f64026a3974a"}
90962024-02-20T22:45:22.404Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:21.921107388Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90972024-02-20T22:45:22.404Zhostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474}
90982024-02-20T22:45:22.404ZLoading write count information from file "/tmp/test_repair_verify.data"
90992024-02-20T22:45:22.404ZRun Repair workload
91002024-02-20T22:45:22.414Z01/30 Write block 279 len 40960 data: 3 3 6 6 8 8 8 9 9 8
91012024-02-20T22:45:22.414Z02/30 Write block 203 len 40960 data: 5 6 7 7 7 6 6 6 6 6
91022024-02-20T22:45:22.418Z03/30 Write block 182 len 32768 data: 9 6 7 7 6 6 6 6
91032024-02-20T22:45:22.418Z04/30 Write block 150 len 20480 data: 11 12 10 6 6
91042024-02-20T22:45:22.419Z05/30 Write block 215 len 12288 data: 3 3 2
91052024-02-20T22:45:22.419Z06/30 Read block 475 len 4096
91062024-02-20T22:45:22.419Z07/30 Write block 491 len 24576 data: 5 8 10 10 10 10
91072024-02-20T22:45:22.419Z08/30 Write block 230 len 32768 data: 13 12 13 15 14 10 10 10
91082024-02-20T22:45:22.419Z09/30 Write block 454 len 24576 data: 9 10 12 13 14 12
91092024-02-20T22:45:22.419Z10/30 Write block 234 len 24576 data: 15 11 11 11 7 5
91102024-02-20T22:45:22.419Z11/30 Write block 203 len 36864 data: 6 7 8 8 8 7 7 7 7
91112024-02-20T22:45:22.419Z12/30 Read block 513 len 8192
91122024-02-20T22:45:22.419Z13/30 Read block 141 len 32768
91132024-02-20T22:45:22.419Z14/30 Read block 342 len 4096
91142024-02-20T22:45:22.419Z15/30 Read block 531 len 20480
91152024-02-20T22:45:22.419Z16/30 Read block 174 len 20480
91162024-02-20T22:45:22.419Z17/30 Write block 517 len 20480 data: 6 4 3 4 5
91172024-02-20T22:45:22.419Z18/30 Flush
91182024-02-20T22:45:22.419Z19/30 Write block 396 len 8192 data: 10 12
91192024-02-20T22:45:22.419Z20/30 Write block 181 len 16384 data: 9 10 7 8
91202024-02-20T22:45:22.419Z21/30 Flush
91212024-02-20T22:45:22.419Z22/30 Write block 456 len 12288 data: 13 14 15
91222024-02-20T22:45:22.419Z23/30 Flush
91232024-02-20T22:45:22.424Z24/30 Flush
91242024-02-20T22:45:22.424Z25/30 Write block 220 len 16384 data: 6 6 6 6
91252024-02-20T22:45:22.424Z26/30 Write block 517 len 20480 data: 7 5 4 5 6
91262024-02-20T22:45:22.424Z27/30 Read block 462 len 32768
91272024-02-20T22:45:22.424Z28/30 Read block 450 len 40960
91282024-02-20T22:45:22.429Z29/30 Read block 179 len 20480
91292024-02-20T22:45:22.429Z30/30 Read block 405 len 20480
91302024-02-20T22:45:22.429Z----------------------------------------------------------------
91312024-02-20T22:45:22.429Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91322024-02-20T22:45:22.429ZStates: Active Active Active
91332024-02-20T22:45:22.429ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91342024-02-20T22:45:22.429Z 1 Acked 1000 Write 10 Done Done Done false
91352024-02-20T22:45:22.429Z 2 Acked 1001 Write 10 Done Sent Done false
91362024-02-20T22:45:22.429Z 3 Acked 1002 Write 8 Done Sent Done false
91372024-02-20T22:45:22.429Z 4 Acked 1003 Write 5 Done Sent Done false
91382024-02-20T22:45:22.429Z 5 Acked 1004 Write 3 Done Sent Done false
91392024-02-20T22:45:22.429Z 6 Acked 1005 Read 1 Done Sent Done false
91402024-02-20T22:45:22.429Z 7 Acked 1006 Write 6 Done Sent Done false
91412024-02-20T22:45:22.429Z 8 Acked 1007 Write 8 Done Sent Done false
91422024-02-20T22:45:22.429Z 9 Acked 1008 Write 6 Done Sent Done false
91432024-02-20T22:45:22.430Z 10 Acked 1009 Write 6 Done Sent Done false
91442024-02-20T22:45:22.430Z 11 Acked 1010 Write 9 Done Sent Done false
91452024-02-20T22:45:22.430Z 12 Acked 1011 Read 2 Done Sent Done false
91462024-02-20T22:45:22.430Z 13 Acked 1012 Read 8 Done Sent Done false
91472024-02-20T22:45:22.430Z 14 Acked 1013 Read 1 Done Sent Done false
91482024-02-20T22:45:22.430Z 15 Acked 1014 Read 5 Done Sent Done false
91492024-02-20T22:45:22.430Z 16 Acked 1015 Read 5 Done Sent Done false
91502024-02-20T22:45:22.430Z 17 Acked 1016 Write 5 Done Sent Done false
91512024-02-20T22:45:22.430Z 18 Acked 1017 Flush 0 Done Sent Done false
91522024-02-20T22:45:22.430Z 19 Acked 1018 Write 2 Done Sent Done false
91532024-02-20T22:45:22.430Z 20 Acked 1019 Write 4 Done Sent Done false
91542024-02-20T22:45:22.430Z 21 Acked 1020 Flush 0 Done Sent Done false
91552024-02-20T22:45:22.430Z 22 Acked 1021 Write 3 Done Sent Done false
91562024-02-20T22:45:22.430Z 23 Acked 1022 Flush 0 Done Sent Done false
91572024-02-20T22:45:22.430Z 24 Acked 1023 Flush 0 Done Sent Done false
91582024-02-20T22:45:22.430Z 25 Acked 1024 Write 4 Done Sent Done false
91592024-02-20T22:45:22.430Z 26 Acked 1025 Write 5 Done Sent Done false
91602024-02-20T22:45:22.430Z 27 Acked 1026 Read 8 Done Sent Done false
91612024-02-20T22:45:22.430Z 28 Acked 1027 Read 10 Done Sent Done false
91622024-02-20T22:45:22.430Z 29 Acked 1028 Read 5 Done Sent Done false
91632024-02-20T22:45:22.430Z 30 Acked 1029 Read 5 Done Sent Sent false
91642024-02-20T22:45:22.430Z STATES DS:0 DS:1 DS:2 TOTAL
91652024-02-20T22:45:22.430Z New 0 0 0 0
91662024-02-20T22:45:22.430Z Sent 0 29 1 30
91672024-02-20T22:45:22.430Z Done 30 1 29 60
91682024-02-20T22:45:22.431Z Skipped 0 0 0 0
91692024-02-20T22:45:22.431Z Error 0 0 0 0
91702024-02-20T22:45:22.431ZLast Flush: 1023 0 1023
91712024-02-20T22:45:22.431ZDownstairs last five completed:
91722024-02-20T22:45:22.431ZUpstairs last five completed: 30 29 28 27 26
91732024-02-20T22:45:22.431Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:21.951192411Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1474}
91742024-02-20T22:45:22.431ZWrote out file "/tmp/test_repair_verify.data"
91752024-02-20T22:45:22.435Z
91762024-02-20T22:45:22.435ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91772024-02-20T22:45:22.449ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91782024-02-20T22:45:22.449Z 0 000-019 74 74 74 133 133 133 F F F
91792024-02-20T22:45:22.449Z 1 020-039 81 81 81 142 142 142 F F F
91802024-02-20T22:45:22.450Z 2 040-059 81 81 81 142 142 142 F F F
91812024-02-20T22:45:22.450Z 3 060-079 77 77 77 136 136 136 F F F
91822024-02-20T22:45:22.450Z 4 080-099 81 81 81 142 142 142 F F F
91832024-02-20T22:45:22.450Z 5 100-119 81 81 81 142 142 142 F F F
91842024-02-20T22:45:22.454Z 6 120-139 74 74 74 133 133 133 F F F
91852024-02-20T22:45:22.454Z 7 140-159 84 82 84 146 143 146 F F F <---
91862024-02-20T22:45:22.454Z 8 160-179 81 81 81 142 142 142 F F F
91872024-02-20T22:45:22.454Z 9 180-199 84 78 84 147 138 147 F F F <---
91882024-02-20T22:45:22.454Z 10 200-219 84 82 84 146 143 146 F F F <---
91892024-02-20T22:45:22.454Z 11 220-239 84 82 84 146 143 146 T F T <---
91902024-02-20T22:45:22.454Z 12 240-259 81 81 81 142 142 142 F F F
91912024-02-20T22:45:22.454Z 13 260-279 84 81 84 146 142 146 F T F <---
91922024-02-20T22:45:22.454Z 14 280-299 84 77 84 146 136 146 F T F <---
91932024-02-20T22:45:22.454Z 15 300-319 81 81 81 142 142 142 F F F
91942024-02-20T22:45:22.454Z 16 320-339 81 81 81 142 142 142 F F F
91952024-02-20T22:45:22.455Z 17 340-359 81 81 81 142 142 142 F F F
91962024-02-20T22:45:22.455Z 18 360-379 75 75 75 134 134 134 F F F
91972024-02-20T22:45:22.455Z 19 380-399 84 78 84 147 137 147 F F F <---
91982024-02-20T22:45:22.455Z 20 400-419 83 83 83 145 145 145 F F F
91992024-02-20T22:45:22.455Z 21 420-439 83 83 83 145 145 145 F F F
92002024-02-20T22:45:22.455Z 22 440-459 84 81 84 148 142 148 F F F <---
92012024-02-20T22:45:22.455Z 23 460-479 80 80 80 140 140 140 F F F
92022024-02-20T22:45:22.455Z 24 480-499 84 80 84 146 140 146 F F F <---
92032024-02-20T22:45:22.455Z 25 500-519 84 81 84 146 142 146 T F T <---
92042024-02-20T22:45:22.455Z 26 520-539 84 83 84 146 145 146 T F T <---
92052024-02-20T22:45:22.455Z 27 540-559 81 81 81 142 142 142 F F F
92062024-02-20T22:45:22.455Z 28 560-579 82 82 82 144 144 144 F F F
92072024-02-20T22:45:22.455Z 29 580-599 83 83 83 145 145 145 F F F
92082024-02-20T22:45:22.455ZMax gen: 84, Max flush: 148
92092024-02-20T22:45:22.455ZError: Difference in extent metadata found!
92102024-02-20T22:45:22.455ZA Difference in extent metadata is expected here
92112024-02-20T22:45:22.455ZOn loop 42
92122024-02-20T22:45:22.455Z
92132024-02-20T22:45:22.460ZVerifying data now
92142024-02-20T22:45:27.002ZNo change after verify
92152024-02-20T22:45:27.002ZLoop: 42 Downstairs dump after verify (and repair):
92162024-02-20T22:45:27.016ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92172024-02-20T22:45:27.016Z 0 000-019 74 74 74 133 133 133 F F F
92182024-02-20T22:45:27.016Z 1 020-039 81 81 81 142 142 142 F F F
92192024-02-20T22:45:27.016Z 2 040-059 81 81 81 142 142 142 F F F
92202024-02-20T22:45:27.016Z 3 060-079 77 77 77 136 136 136 F F F
92212024-02-20T22:45:27.016Z 4 080-099 81 81 81 142 142 142 F F F
92222024-02-20T22:45:27.016Z 5 100-119 81 81 81 142 142 142 F F F
92232024-02-20T22:45:27.016Z 6 120-139 74 74 74 133 133 133 F F F
92242024-02-20T22:45:27.016Z 7 140-159 84 84 84 146 146 146 F F F
92252024-02-20T22:45:27.016Z 8 160-179 81 81 81 142 142 142 F F F
92262024-02-20T22:45:27.021Z 9 180-199 84 84 84 147 147 147 F F F
92272024-02-20T22:45:27.021Z 10 200-219 84 84 84 146 146 146 F F F
92282024-02-20T22:45:27.021Z 11 220-239 85 85 85 149 149 149 F F F
92292024-02-20T22:45:27.021Z 12 240-259 81 81 81 142 142 142 F F F
92302024-02-20T22:45:27.021Z 13 260-279 84 84 84 146 146 146 F F F
92312024-02-20T22:45:27.021Z 14 280-299 84 84 84 146 146 146 F F F
92322024-02-20T22:45:27.021Z 15 300-319 81 81 81 142 142 142 F F F
92332024-02-20T22:45:27.021Z 16 320-339 81 81 81 142 142 142 F F F
92342024-02-20T22:45:27.021Z 17 340-359 81 81 81 142 142 142 F F F
92352024-02-20T22:45:27.021Z 18 360-379 75 75 75 134 134 134 F F F
92362024-02-20T22:45:27.021Z 19 380-399 84 84 84 147 147 147 F F F
92372024-02-20T22:45:27.021Z 20 400-419 83 83 83 145 145 145 F F F
92382024-02-20T22:45:27.021Z 21 420-439 83 83 83 145 145 145 F F F
92392024-02-20T22:45:27.021Z 22 440-459 84 84 84 148 148 148 F F F
92402024-02-20T22:45:27.021Z 23 460-479 80 80 80 140 140 140 F F F
92412024-02-20T22:45:27.021Z 24 480-499 84 84 84 146 146 146 F F F
92422024-02-20T22:45:27.021Z 25 500-519 85 85 85 149 149 149 F F F
92432024-02-20T22:45:27.021Z 26 520-539 85 85 85 149 149 149 F F F
92442024-02-20T22:45:27.022Z 27 540-559 81 81 81 142 142 142 F F F
92452024-02-20T22:45:27.022Z 28 560-579 82 82 82 144 144 144 F F F
92462024-02-20T22:45:27.022Z 29 580-599 83 83 83 145 145 145 F F F
92472024-02-20T22:45:27.022ZMax gen: 85, Max flush: 149
92482024-02-20T22:45:27.022Z
92492024-02-20T22:45:27.022ZBegin loop 43
92502024-02-20T22:45:27.022ZDownstairs to restart: 1
92512024-02-20T22:45:27.039Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.563437442Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482}
92522024-02-20T22:45:27.039Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.563728312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482}
92532024-02-20T22:45:27.047Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.56376504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482}
92542024-02-20T22:45:27.047Z{"msg":"Crucible 0abbae8b-4afb-4ef1-8198-15f2ee8ed347 has session id: ac58c026-1b28-40bc-80f9-f861768b1e3a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.563805589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92552024-02-20T22:45:27.048Z{"msg":"Upstairs opts: Upstairs UUID: 0abbae8b-4afb-4ef1-8198-15f2ee8ed347, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
92562024-02-20T22:45:27.048Z,"time":"2024-02-20T22:45:26.563850528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92572024-02-20T22:45:27.048Z{"msg":"Crucible stats registered with UUID: 0abbae8b-4afb-4ef1-8198-15f2ee8ed347","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.563892066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92582024-02-20T22:45:27.048Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.563940424Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482}
92592024-02-20T22:45:27.048Z{"msg":"0abbae8b-4afb-4ef1-8198-15f2ee8ed347 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564109028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92602024-02-20T22:45:27.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564159886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92612024-02-20T22:45:27.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564195365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92622024-02-20T22:45:27.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564240883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92632024-02-20T22:45:27.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564285472Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92642024-02-20T22:45:27.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564319781Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92652024-02-20T22:45:27.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564358609Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92662024-02-20T22:45:27.048Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564384738Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92672024-02-20T22:45:27.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564431247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92682024-02-20T22:45:27.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564460336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92692024-02-20T22:45:27.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564555292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92702024-02-20T22:45:27.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.564594491Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92712024-02-20T22:45:27.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.56462168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92722024-02-20T22:45:27.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565271247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92732024-02-20T22:45:27.048Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565317525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92742024-02-20T22:45:27.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565366073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92752024-02-20T22:45:27.048Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565396922Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92762024-02-20T22:45:27.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565437691Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92772024-02-20T22:45:27.048Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565492369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92782024-02-20T22:45:27.049Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565922083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92792024-02-20T22:45:27.049Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.565978481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92802024-02-20T22:45:27.049Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566108297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92812024-02-20T22:45:27.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.56658841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92822024-02-20T22:45:27.049Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566652907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92832024-02-20T22:45:27.049Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566693446Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92842024-02-20T22:45:27.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566713225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92852024-02-20T22:45:27.049Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566745144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92862024-02-20T22:45:27.049Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566766463Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92872024-02-20T22:45:27.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566794372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,Wait for a query_work_queue command to finish before sending IO
92882024-02-20T22:45:27.049Z"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92892024-02-20T22:45:27.049Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566818151Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92902024-02-20T22:45:27.049Z{"msg":"[0]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 146, 142, 147, 146, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.56685489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92912024-02-20T22:45:27.049Z{"msg":"[0]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 84, 81, 84, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566871349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92922024-02-20T22:45:27.049Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566914988Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92932024-02-20T22:45:27.049Z{"msg":"[1]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 146, 142, 147, 146, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566937947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92942024-02-20T22:45:27.049Z{"msg":"[1]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 84, 81, 84, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566967906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92952024-02-20T22:45:27.049Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.566984825Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92962024-02-20T22:45:27.049Z{"msg":"[2]R flush_numbers[0..12]: [133, 142, 142, 136, 142, 142, 133, 146, 142, 147, 146, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567017654Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92972024-02-20T22:45:27.049Z{"msg":"[2]R generation[0..12]: [74, 81, 81, 77, 81, 81, 74, 84, 81, 84, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567037883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92982024-02-20T22:45:27.049Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567068092Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
92992024-02-20T22:45:27.049Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567083382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93002024-02-20T22:45:27.049Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567114981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93012024-02-20T22:45:27.049Z{"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.56713056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93022024-02-20T22:45:27.049Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567161089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93032024-02-20T22:45:27.050Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567180538Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93042024-02-20T22:45:27.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93052024-02-20T22:45:27.050Z,"time":"2024-02-20T22:45:26.567208017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93062024-02-20T22:45:27.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567224497Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93072024-02-20T22:45:27.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567256466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93082024-02-20T22:45:27.050Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567273415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93092024-02-20T22:45:27.050Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567302624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93102024-02-20T22:45:27.050Z{"msg":"0abbae8b-4afb-4ef1-8198-15f2ee8ed347 is now active with session: ac58c026-1b28-40bc-80f9-f861768b1e3a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567317683Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93112024-02-20T22:45:27.050Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567346862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482,"session_id":"ac58c026-1b28-40bc-80f9-f861768b1e3a"}
93122024-02-20T22:45:27.050Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:26.567363882Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482}
93132024-02-20T22:45:27.050ZLoading write count information from file "/tmp/test_repair_verify.data"
93142024-02-20T22:45:27.050ZRun Repair workload
93152024-02-20T22:45:27.050Z01/30 Write block 3 len 36864 data: 6 7 8 6 5 5 5 7 6
93162024-02-20T22:45:27.055Z02/30 Read block 21 len 8192
93172024-02-20T22:45:27.055Z03/30 Read block 562 len 24576
93182024-02-20T22:45:27.055Z04/30 Read block 547 len 12288
93192024-02-20T22:45:27.055Z05/30 Read block 242 len 32768
93202024-02-20T22:45:27.055Z06/30 Write block 402 len 28672 data: 8 9 9 9 9 9 8
93212024-02-20T22:45:27.055Z07/30 Read block 131 len 20480
93222024-02-20T22:45:27.055Z08/30 Read block 368 len 12288
93232024-02-20T22:45:27.055Z09/30 Write block 43 len 24576 data: 8 7 5 5 6 7
93242024-02-20T22:45:27.055Z10/30 Write block 488 len 12288 data: 6 4 4
93252024-02-20T22:45:27.055Z11/30 Read block 322 len 36864
93262024-02-20T22:45:27.055Z12/30 Read block 349 len 4096
93272024-02-20T22:45:27.059Z13/30 Write block 127 len 28672 data: 7 7 7 7 7 7 8
93282024-02-20T22:45:27.059Z14/30 Write block 340 len 24576 data: 8 9 9 8 5 5
93292024-02-20T22:45:27.059Z15/30 Read block 475 len 40960
93302024-02-20T22:45:27.059Z16/30 Read block 574 len 40960
93312024-02-20T22:45:27.060Z17/30 Write block 19 len 16384 data: 6 6 5 6
93322024-02-20T22:45:27.060Z18/30 Read block 29 len 36864
93332024-02-20T22:45:27.064Z19/30 Read block 153 len 36864
93342024-02-20T22:45:27.064Z20/30 Write block 342 len 8192 data: 10 9
93352024-02-20T22:45:27.064Z21/30 Read block 553 len 8192
93362024-02-20T22:45:27.064Z22/30 Read block 251 len 36864
93372024-02-20T22:45:27.064Z23/30 Read block 78 len 24576
93382024-02-20T22:45:27.069Z24/30 Read block 183 len 28672
93392024-02-20T22:45:27.069Z25/30 Read block 519 len 20480
93402024-02-20T22:45:27.069Z26/30 Write block 361 len 24576 data: 10 11 14 13 15 13
93412024-02-20T22:45:27.069Z27/30 Read block 400 len 16384
93422024-02-20T22:45:27.069Z28/30 Read block 137 len 12288
93432024-02-20T22:45:27.074Z29/30 Write block 473 len 16384 data: 9 10 10 10
93442024-02-20T22:45:27.074Z30/30 Read block 396 len 28672
93452024-02-20T22:45:27.074Z----------------------------------------------------------------
93462024-02-20T22:45:27.074Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93472024-02-20T22:45:27.074ZStates: Active Active Active
93482024-02-20T22:45:27.074ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93492024-02-20T22:45:27.074Z 1 Acked 1000 Write 9 Done Done Done false
93502024-02-20T22:45:27.075Z 2 Acked 1001 Read 2 Done Done Done false
93512024-02-20T22:45:27.075Z 3 Acked 1002 Read 6 Done Done Done false
93522024-02-20T22:45:27.075Z 4 Acked 1003 Read 3 Done Done Done false
93532024-02-20T22:45:27.075Z 5 Acked 1004 Read 8 Done Done Done false
93542024-02-20T22:45:27.075Z 6 Acked 1005 Write 7 Done Done Done false
93552024-02-20T22:45:27.075Z 7 Acked 1006 Read 5 Done Done Done false
93562024-02-20T22:45:27.075Z 8 Acked 1007 Read 3 Done Sent Done false
93572024-02-20T22:45:27.075Z 9 Acked 1008 Write 6 Done Sent Done false
93582024-02-20T22:45:27.075Z 10 Acked 1009 Write 3 Done Sent Done false
93592024-02-20T22:45:27.075Z 11 Acked 1010 Read 9 Done Sent Done false
93602024-02-20T22:45:27.075Z 12 Acked 1011 Read 1 Done Sent Done false
93612024-02-20T22:45:27.075Z 13 Acked 1012 Write 7 Done Sent Done false
93622024-02-20T22:45:27.075Z 14 Acked 1013 Write 6 Done Sent Done false
93632024-02-20T22:45:27.075Z 15 Acked 1014 Read 10 Done Sent Done false
93642024-02-20T22:45:27.075Z 16 Acked 1015 Read 10 Done Sent Done false
93652024-02-20T22:45:27.075Z 17 Acked 1016 Write 4 Done Sent Done false
93662024-02-20T22:45:27.075Z 18 Acked 1017 Read 9 Done Sent Done false
93672024-02-20T22:45:27.075Z 19 Acked 1018 Read 9 Done Sent Done false
93682024-02-20T22:45:27.075Z 20 Acked 1019 Write 2 Done Sent Done false
93692024-02-20T22:45:27.075Z 21 Acked 1020 Read 2 Done Sent Done false
93702024-02-20T22:45:27.075Z 22 Acked 1021 Read 9 Done Sent Done false
93712024-02-20T22:45:27.075Z 23 Acked 1022 Read 6 Done Sent Done false
93722024-02-20T22:45:27.075Z 24 Acked 1023 Read 7 Done Sent Done false
93732024-02-20T22:45:27.075Z 25 Acked 1024 Read 5 Done Sent Done false
93742024-02-20T22:45:27.076Z 26 Acked 1025 Write 6 Done Sent Done false
93752024-02-20T22:45:27.076Z 27 Acked 1026 Read 4 Done Sent Done false
93762024-02-20T22:45:27.076Z 28 Acked 1027 Read 3 Done Sent Done false
93772024-02-20T22:45:27.076Z 29 Acked 1028 Write 4 Done Sent Done false
93782024-02-20T22:45:27.076Z 30 Acked 1029 Read 7 Sent Sent Done false
93792024-02-20T22:45:27.076Z STATES DS:0 DS:1 DS:2 TOTAL
93802024-02-20T22:45:27.076Z New 0 0 0 0
93812024-02-20T22:45:27.076Z Sent 1 23 0 24
93822024-02-20T22:45:27.076Z Done 29 7 30 66
93832024-02-20T22:45:27.076Z Skipped 0 0 0 0
93842024-02-20T22:45:27.076Z Error 0 0 0 0
93852024-02-20T22:45:27.076ZLast Flush: 0 0 0
93862024-02-20T22:45:27.076ZDownstairs last five completed:
93872024-02-20T22:45:27.076ZUpstairs last five completed: 30 29 28 27 26
93882024-02-20T22:45:27.076Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:26.59563755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1482}
93892024-02-20T22:45:27.076ZWrote out file "/tmp/test_repair_verify.data"
93902024-02-20T22:45:27.081Z
93912024-02-20T22:45:27.081ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93922024-02-20T22:45:27.096ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93932024-02-20T22:45:27.096Z 0 000-019 74 74 74 133 133 133 T T T
93942024-02-20T22:45:27.096Z 1 020-039 81 81 81 142 142 142 T F T <---
93952024-02-20T22:45:27.096Z 2 040-059 81 81 81 142 142 142 T F T <---
93962024-02-20T22:45:27.096Z 3 060-079 77 77 77 136 136 136 F F F
93972024-02-20T22:45:27.096Z 4 080-099 81 81 81 142 142 142 F F F
93982024-02-20T22:45:27.096Z 5 100-119 81 81 81 142 142 142 F F F
93992024-02-20T22:45:27.096Z 6 120-139 74 74 74 133 133 133 T F T <---
94002024-02-20T22:45:27.096Z 7 140-159 84 84 84 146 146 146 F F F
94012024-02-20T22:45:27.096Z 8 160-179 81 81 81 142 142 142 F F F
94022024-02-20T22:45:27.096Z 9 180-199 84 84 84 147 147 147 F F F
94032024-02-20T22:45:27.096Z 10 200-219 84 84 84 146 146 146 F F F
94042024-02-20T22:45:27.101Z 11 220-239 85 85 85 149 149 149 F F F
94052024-02-20T22:45:27.101Z 12 240-259 81 81 81 142 142 142 F F F
94062024-02-20T22:45:27.101Z 13 260-279 84 84 84 146 146 146 F F F
94072024-02-20T22:45:27.101Z 14 280-299 84 84 84 146 146 146 F F F
94082024-02-20T22:45:27.101Z 15 300-319 81 81 81 142 142 142 F F F
94092024-02-20T22:45:27.101Z 16 320-339 81 81 81 142 142 142 F F F
94102024-02-20T22:45:27.101Z 17 340-359 81 81 81 142 142 142 T F T <---
94112024-02-20T22:45:27.101Z 18 360-379 75 75 75 134 134 134 T F T <---
94122024-02-20T22:45:27.101Z 19 380-399 84 84 84 147 147 147 F F F
94132024-02-20T22:45:27.101Z 20 400-419 83 83 83 145 145 145 T T T
94142024-02-20T22:45:27.101Z 21 420-439 83 83 83 145 145 145 F F F
94152024-02-20T22:45:27.101Z 22 440-459 84 84 84 148 148 148 F F F
94162024-02-20T22:45:27.101Z 23 460-479 80 80 80 140 140 140 T F T <---
94172024-02-20T22:45:27.101Z 24 480-499 84 84 84 146 146 146 T F T <---
94182024-02-20T22:45:27.101Z 25 500-519 85 85 85 149 149 149 F F F
94192024-02-20T22:45:27.101Z 26 520-539 85 85 85 149 149 149 F F F
94202024-02-20T22:45:27.101Z 27 540-559 81 81 81 142 142 142 F F F
94212024-02-20T22:45:27.101Z 28 560-579 82 82 82 144 144 144 F F F
94222024-02-20T22:45:27.101Z 29 580-599 83 83 83 145 145 145 F F F
94232024-02-20T22:45:27.101ZMax gen: 85, Max flush: 149
94242024-02-20T22:45:27.101ZError: Difference in extent metadata found!
94252024-02-20T22:45:27.101ZA Difference in extent metadata is expected here
94262024-02-20T22:45:27.101ZOn loop 43
94272024-02-20T22:45:27.101Z
94282024-02-20T22:45:27.106ZVerifying data now
94292024-02-20T22:45:31.574ZNo change after verify
94302024-02-20T22:45:31.574ZLoop: 43 Downstairs dump after verify (and repair):
94312024-02-20T22:45:31.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94322024-02-20T22:45:31.588Z 0 000-019 86 86 86 150 150 150 F F F
94332024-02-20T22:45:31.588Z 1 020-039 86 86 86 150 150 150 F F F
94342024-02-20T22:45:31.588Z 2 040-059 86 86 86 150 150 150 F F F
94352024-02-20T22:45:31.588Z 3 060-079 77 77 77 136 136 136 F F F
94362024-02-20T22:45:31.588Z 4 080-099 81 81 81 142 142 142 F F F
94372024-02-20T22:45:31.588Z 5 100-119 81 81 81 142 142 142 F F F
94382024-02-20T22:45:31.588Z 6 120-139 86 86 86 150 150 150 F F F
94392024-02-20T22:45:31.593Z 7 140-159 84 84 84 146 146 146 F F F
94402024-02-20T22:45:31.593Z 8 160-179 81 81 81 142 142 142 F F F
94412024-02-20T22:45:31.593Z 9 180-199 84 84 84 147 147 147 F F F
94422024-02-20T22:45:31.593Z 10 200-219 84 84 84 146 146 146 F F F
94432024-02-20T22:45:31.593Z 11 220-239 85 85 85 149 149 149 F F F
94442024-02-20T22:45:31.593Z 12 240-259 81 81 81 142 142 142 F F F
94452024-02-20T22:45:31.593Z 13 260-279 84 84 84 146 146 146 F F F
94462024-02-20T22:45:31.593Z 14 280-299 84 84 84 146 146 146 F F F
94472024-02-20T22:45:31.593Z 15 300-319 81 81 81 142 142 142 F F F
94482024-02-20T22:45:31.593Z 16 320-339 81 81 81 142 142 142 F F F
94492024-02-20T22:45:31.593Z 17 340-359 86 86 86 150 150 150 F F F
94502024-02-20T22:45:31.594Z 18 360-379 86 86 86 150 150 150 F F F
94512024-02-20T22:45:31.594Z 19 380-399 84 84 84 147 147 147 F F F
94522024-02-20T22:45:31.594Z 20 400-419 86 86 86 150 150 150 F F F
94532024-02-20T22:45:31.594Z 21 420-439 83 83 83 145 145 145 F F F
94542024-02-20T22:45:31.594Z 22 440-459 84 84 84 148 148 148 F F F
94552024-02-20T22:45:31.594Z 23 460-479 86 86 86 150 150 150 F F F
94562024-02-20T22:45:31.594Z 24 480-499 86 86 86 150 150 150 F F F
94572024-02-20T22:45:31.594Z 25 500-519 85 85 85 149 149 149 F F F
94582024-02-20T22:45:31.594Z 26 520-539 85 85 85 149 149 149 F F F
94592024-02-20T22:45:31.594Z 27 540-559 81 81 81 142 142 142 F F F
94602024-02-20T22:45:31.594Z 28 560-579 82 82 82 144 144 144 F F F
94612024-02-20T22:45:31.594Z 29 580-599 83 83 83 145 145 145 F F F
94622024-02-20T22:45:31.594ZMax gen: 86, Max flush: 150
94632024-02-20T22:45:31.594Z
94642024-02-20T22:45:31.594ZBegin loop 44
94652024-02-20T22:45:31.594ZDownstairs to restart: 2
94662024-02-20T22:45:31.611Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.135452153Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490}
94672024-02-20T22:45:31.618Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.135791531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490}
94682024-02-20T22:45:31.618Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":Crucible runtime is spawned
94692024-02-20T22:45:31.618Z30,"time":"2024-02-20T22:45:31.13584187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490}
94702024-02-20T22:45:31.618Z{"msg":"Crucible f725eed5-49d3-409a-a959-1f93a260351a has session id: 476d1437-18db-4d47-88ef-51c1d69c0ed8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.135886808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94712024-02-20T22:45:31.619Z{"msg":"Upstairs opts: Upstairs UUID: f725eed5-49d3-409a-a959-1f93a260351a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.135945466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94722024-02-20T22:45:31.619Z{"msg":"Crucible stats registered with UUID: f725eed5-49d3-409a-a959-1f93a260351a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.135975995Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94732024-02-20T22:45:31.619Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136014483Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490}
94742024-02-20T22:45:31.619Z{"msg":"f725eed5-49d3-409a-a959-1f93a260351a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136039912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94752024-02-20T22:45:31.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136080191Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94762024-02-20T22:45:31.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.13610158Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94772024-02-20T22:45:31.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136126239Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94782024-02-20T22:45:31.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136153388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94792024-02-20T22:45:31.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136184627Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94802024-02-20T22:45:31.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136207646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94812024-02-20T22:45:31.619Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136230996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94822024-02-20T22:45:31.619Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136278554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94832024-02-20T22:45:31.619Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136322152Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94842024-02-20T22:45:31.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136360591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94852024-02-20T22:45:31.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.13639555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94862024-02-20T22:45:31.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.136422589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94872024-02-20T22:45:31.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137094055Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94882024-02-20T22:45:31.619Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137140183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94892024-02-20T22:45:31.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137166992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94902024-02-20T22:45:31.619Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137203941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94912024-02-20T22:45:31.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.13722694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94922024-02-20T22:45:31.619Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137252099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94932024-02-20T22:45:31.619Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137655295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94942024-02-20T22:45:31.619Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137692603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94952024-02-20T22:45:31.619Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.137971673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94962024-02-20T22:45:31.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138161227Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94972024-02-20T22:45:31.620Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138213905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94982024-02-20T22:45:31.620Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138260653Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
94992024-02-20T22:45:31.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138296912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95002024-02-20T22:45:31.620Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138314021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95012024-02-20T22:45:31.620Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.13832803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95022024-02-20T22:45:31.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138452486Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95032024-02-20T22:45:31.620Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138519364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95042024-02-20T22:45:31.620Z{"msg":"[0]R flush_numbers[0..12]: [150, 150, 150, 136, 142, 142, 150, 146, 142, 147, 146, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138555993Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95052024-02-20T22:45:31.620Z{"msg":"[0]R generation[0..12]: [86, 86, 86, 77, 81, 81, 86, 84, 81, 84, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138582281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95062024-02-20T22:45:31.620Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.13861608Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95072024-02-20T22:45:31.620ZWait for a query_work_queue command to finish before sending IO
95082024-02-20T22:45:31.620Z{"msg":"[1]R flush_numbers[0..12]: [150, 150, 150, 136, 142, 142, 150, 146, 142, 147, 146, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138648029Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95092024-02-20T22:45:31.620Z{"msg":"[1]R generation[0..12]: [86, 86, 86, 77, 81, 81, 86, 84, 81, 84, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138680868Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95102024-02-20T22:45:31.620Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138706937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95112024-02-20T22:45:31.620Z{"msg":"[2]R flush_numbers[0..12]: [150, 150, 150, 136, 142, 142, 150, 146, 142, 147, 146, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138740706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95122024-02-20T22:45:31.620Z{"msg":"[2]R generation[0..12]: [86, 86, 86, 77, 81, 81, 86, 84, 81, 84, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138767285Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95132024-02-20T22:45:31.620Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138800024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95142024-02-20T22:45:31.620Z{"msg":"Max found gen is 87","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138831592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95152024-02-20T22:45:31.620Z{"msg":"Generation requested: 88 >= found:87","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138856552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95162024-02-20T22:45:31.620Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138884301Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95172024-02-20T22:45:31.620Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.13890761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95182024-02-20T22:45:31.620Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138935299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95192024-02-20T22:45:31.620Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138957368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95202024-02-20T22:45:31.620Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.138985807Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95212024-02-20T22:45:31.621Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.139032905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95222024-02-20T22:45:31.621Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.139058815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95232024-02-20T22:45:31.621Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.139081474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95242024-02-20T22:45:31.621Z{"msg":"f725eed5-49d3-409a-a959-1f93a260351a is now active with session: 476d1437-18db-4d47-88ef-51c1d69c0ed8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.139104273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95252024-02-20T22:45:31.621ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95262024-02-20T22:45:31.621Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.139134142Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490,"session_id":"476d1437-18db-4d47-88ef-51c1d69c0ed8"}
95272024-02-20T22:45:31.621Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:31.139151221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490}
95282024-02-20T22:45:31.621ZLoading write count information from file "/tmp/test_repair_verify.data"
95292024-02-20T22:45:31.621ZRun Repair workload
95302024-02-20T22:45:31.621Z01/30 Write block 580 len 36864 data: 12 10 10 9 7 7 7 7 8
95312024-02-20T22:45:31.626Z02/30 Write block 382 len 24576 data: 5 4 5 5 4 4
95322024-02-20T22:45:31.626Z03/30 Read block 580 len 32768
95332024-02-20T22:45:31.626Z04/30 Write block 23 len 40960 data: 6 6 11 10 13 14 16 14 15 14
95342024-02-20T22:45:31.626Z05/30 Read block 489 len 12288
95352024-02-20T22:45:31.626Z06/30 Write block 101 len 4096 data: 5
95362024-02-20T22:45:31.626Z07/30 Write block 287 len 20480 data: 10 9 8 7 8
95372024-02-20T22:45:31.627Z08/30 Write block 378 len 36864 data: 8 5 5 4 6 5 6 6 5
95382024-02-20T22:45:31.627Z09/30 Read block 155 len 8192
95392024-02-20T22:45:31.627Z10/30 Flush
95402024-02-20T22:45:31.631Z11/30 Write block 44 len 24576 data: 8 6 6 7 8 7
95412024-02-20T22:45:31.631Z12/30 Read block 3 len 28672
95422024-02-20T22:45:31.631Z13/30 Read block 392 len 4096
95432024-02-20T22:45:31.631Z14/30 Read block 547 len 8192
95442024-02-20T22:45:31.631Z15/30 Flush
95452024-02-20T22:45:31.634Z16/30 Write block 415 len 32768 data: 10 8 6 7 9 9 7 8
95462024-02-20T22:45:31.634Z17/30 Write block 471 len 8192 data: 8 9
95472024-02-20T22:45:31.634Z18/30 Write block 371 len 12288 data: 12 11 9
95482024-02-20T22:45:31.635Z19/30 Read block 121 len 16384
95492024-02-20T22:45:31.635Z20/30 Flush
95502024-02-20T22:45:31.638Z21/30 Write block 206 len 8192 data: 9 9
95512024-02-20T22:45:31.639Z22/30 Read block 67 len 36864
95522024-02-20T22:45:31.639Z23/30 Read block 442 len 12288
95532024-02-20T22:45:31.639Z24/30 Read block 540 len 4096
95542024-02-20T22:45:31.645Z25/30 Read block 435 len 20480
95552024-02-20T22:45:31.645Z26/30 Read block 583 len 4096
95562024-02-20T22:45:31.645Z27/30 Read block 545 len 16384
95572024-02-20T22:45:31.645Z28/30 Read block 207 len 8192
95582024-02-20T22:45:31.645Z29/30 Write block 359 len 8192 data: 8 10
95592024-02-20T22:45:31.645Z30/30 Write block 260 len 24576 data: 5 5 5 5 9 9
95602024-02-20T22:45:31.645Z----------------------------------------------------------------
95612024-02-20T22:45:31.645Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95622024-02-20T22:45:31.645ZStates: Active Active Active
95632024-02-20T22:45:31.645ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95642024-02-20T22:45:31.645Z 1 Acked 1000 Write 9 Done Done Done false
95652024-02-20T22:45:31.645Z 2 Acked 1001 Write 6 Done Done Done false
95662024-02-20T22:45:31.645Z 3 Acked 1002 Read 8 Done Done Sent false
95672024-02-20T22:45:31.645Z 4 Acked 1003 Write 10 Done Done Sent false
95682024-02-20T22:45:31.645Z 5 Acked 1004 Read 3 Done Done Sent false
95692024-02-20T22:45:31.645Z 6 Acked 1005 Write 1 Done Done Sent false
95702024-02-20T22:45:31.645Z 7 Acked 1006 Write 5 Done Done Sent false
95712024-02-20T22:45:31.645Z 8 Acked 1007 Write 9 Done Done Sent false
95722024-02-20T22:45:31.645Z 9 Acked 1008 Read 2 Done Done Sent false
95732024-02-20T22:45:31.645Z 10 Acked 1009 Flush 0 Done Done Sent false
95742024-02-20T22:45:31.645Z 11 Acked 1010 Write 6 Done Done Sent false
95752024-02-20T22:45:31.645Z 12 Acked 1011 Read 7 Done Done Sent false
95762024-02-20T22:45:31.646Z 13 Acked 1012 Read 1 Done Done Sent false
95772024-02-20T22:45:31.646Z 14 Acked 1013 Read 2 Done Done Sent false
95782024-02-20T22:45:31.646Z 15 Acked 1014 Flush 0 Done Done Sent false
95792024-02-20T22:45:31.646Z 16 Acked 1015 Write 8 Done Done Sent false
95802024-02-20T22:45:31.646Z 17 Acked 1016 Write 2 Done Done Sent false
95812024-02-20T22:45:31.646Z 18 Acked 1017 Write 3 Done Done Sent false
95822024-02-20T22:45:31.646Z 19 Acked 1018 Read 4 Done Done Sent false
95832024-02-20T22:45:31.646Z 20 Acked 1019 Flush 0 Done Done Sent false
95842024-02-20T22:45:31.646Z 21 Acked 1020 Write 2 Done Done Sent false
95852024-02-20T22:45:31.646Z 22 Acked 1021 Read 9 Done Done Sent false
95862024-02-20T22:45:31.646Z 23 Acked 1022 Read 3 Done Done Sent false
95872024-02-20T22:45:31.646Z 24 Acked 1023 Read 1 Done Done Sent false
95882024-02-20T22:45:31.646Z 25 Acked 1024 Read 5 Done Done Sent false
95892024-02-20T22:45:31.646Z 26 Acked 1025 Read 1 Done Done Sent false
95902024-02-20T22:45:31.646Z 27 Acked 1026 Read 4 Done Done Sent false
95912024-02-20T22:45:31.646Z 28 Acked 1027 Read 2 Done Done Sent false
95922024-02-20T22:45:31.646Z 29 Acked 1028 Write 2 Done Done Sent false
95932024-02-20T22:45:31.646Z 30 Acked 1029 Write 6 Sent Sent Sent false
95942024-02-20T22:45:31.646Z STATES DS:0 DS:1 DS:2 TOTAL
95952024-02-20T22:45:31.646Z New 0 0 0 0
95962024-02-20T22:45:31.646Z Sent 1 1 28 30
95972024-02-20T22:45:31.646Z Done 29 29 2 60
95982024-02-20T22:45:31.646Z Skipped 0 0 0 0
95992024-02-20T22:45:31.646Z Error 0 0 0 0
96002024-02-20T22:45:31.646ZLast Flush: 1019 1019 0
96012024-02-20T22:45:31.647ZDownstairs last five completed:
96022024-02-20T22:45:31.647ZUpstairs last five completed: 30 29 28 27 26
96032024-02-20T22:45:31.647Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:31.167948071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1490}
96042024-02-20T22:45:31.653ZWrote out file "/tmp/test_repair_verify.data"
96052024-02-20T22:45:31.653Z
96062024-02-20T22:45:31.653ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96072024-02-20T22:45:31.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96082024-02-20T22:45:31.668Z 0 000-019 86 86 86 150 150 150 F F F
96092024-02-20T22:45:31.668Z 1 020-039 88 88 86 151 151 150 F F F <---
96102024-02-20T22:45:31.668Z 2 040-059 88 88 86 152 152 150 F F F <---
96112024-02-20T22:45:31.668Z 3 060-079 77 77 77 136 136 136 F F F
96122024-02-20T22:45:31.669Z 4 080-099 81 81 81 142 142 142 F F F
96132024-02-20T22:45:31.669Z 5 100-119 88 88 81 151 151 142 F F F <---
96142024-02-20T22:45:31.669Z 6 120-139 86 86 86 150 150 150 F F F
96152024-02-20T22:45:31.669Z 7 140-159 84 84 84 146 146 146 F F F
96162024-02-20T22:45:31.669Z 8 160-179 81 81 81 142 142 142 F F F
96172024-02-20T22:45:31.669Z 9 180-199 84 84 84 147 147 147 F F F
96182024-02-20T22:45:31.669Z 10 200-219 84 84 84 146 146 146 T T F <---
96192024-02-20T22:45:31.669Z 11 220-239 85 85 85 149 149 149 F F F
96202024-02-20T22:45:31.669Z 12 240-259 81 81 81 142 142 142 F F F
96212024-02-20T22:45:31.674Z 13 260-279 84 84 84 146 146 146 T T F <---
96222024-02-20T22:45:31.674Z 14 280-299 88 88 84 151 151 146 F F F <---
96232024-02-20T22:45:31.674Z 15 300-319 81 81 81 142 142 142 F F F
96242024-02-20T22:45:31.674Z 16 320-339 81 81 81 142 142 142 F F F
96252024-02-20T22:45:31.674Z 17 340-359 86 86 86 150 150 150 T T F <---
96262024-02-20T22:45:31.674Z 18 360-379 88 88 86 153 153 150 T T F <---
96272024-02-20T22:45:31.674Z 19 380-399 88 88 84 151 151 147 F F T <---
96282024-02-20T22:45:31.674Z 20 400-419 88 88 86 153 153 150 F F F <---
96292024-02-20T22:45:31.674Z 21 420-439 88 88 83 153 153 145 F F F <---
96302024-02-20T22:45:31.674Z 22 440-459 84 84 84 148 148 148 F F F
96312024-02-20T22:45:31.674Z 23 460-479 88 88 86 153 153 150 F F F <---
96322024-02-20T22:45:31.674Z 24 480-499 86 86 86 150 150 150 F F F
96332024-02-20T22:45:31.675Z 25 500-519 85 85 85 149 149 149 F F F
96342024-02-20T22:45:31.675Z 26 520-539 85 85 85 149 149 149 F F F
96352024-02-20T22:45:31.675Z 27 540-559 81 81 81 142 142 142 F F F
96362024-02-20T22:45:31.675Z 28 560-579 82 82 82 144 144 144 F F F
96372024-02-20T22:45:31.675Z 29 580-599 88 88 83 151 151 145 F F T <---
96382024-02-20T22:45:31.675ZMax gen: 88, Max flush: 153
96392024-02-20T22:45:31.675ZError: Difference in extent metadata found!
96402024-02-20T22:45:31.675ZA Difference in extent metadata is expected here
96412024-02-20T22:45:31.675ZOn loop 44
96422024-02-20T22:45:31.675Z
96432024-02-20T22:45:31.675ZVerifying data now
96442024-02-20T22:45:36.300ZNo change after verify
96452024-02-20T22:45:36.300ZLoop: 44 Downstairs dump after verify (and repair):
96462024-02-20T22:45:36.312ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96472024-02-20T22:45:36.312Z 0 000-019 86 86 86 150 150 150 F F F
96482024-02-20T22:45:36.312Z 1 020-039 88 88 88 151 151 151 F F F
96492024-02-20T22:45:36.312Z 2 040-059 88 88 88 152 152 152 F F F
96502024-02-20T22:45:36.312Z 3 060-079 77 77 77 136 136 136 F F F
96512024-02-20T22:45:36.312Z 4 080-099 81 81 81 142 142 142 F F F
96522024-02-20T22:45:36.312Z 5 100-119 88 88 88 151 151 151 F F F
96532024-02-20T22:45:36.313Z 6 120-139 86 86 86 150 150 150 F F F
96542024-02-20T22:45:36.313Z 7 140-159 84 84 84 146 146 146 F F F
96552024-02-20T22:45:36.313Z 8 160-179 81 81 81 142 142 142 F F F
96562024-02-20T22:45:36.313Z 9 180-199 84 84 84 147 147 147 F F F
96572024-02-20T22:45:36.313Z 10 200-219 89 89 89 154 154 154 F F F
96582024-02-20T22:45:36.313Z 11 220-239 85 85 85 149 149 149 F F F
96592024-02-20T22:45:36.313Z 12 240-259 81 81 81 142 142 142 F F F
96602024-02-20T22:45:36.313Z 13 260-279 89 89 89 154 154 154 F F F
96612024-02-20T22:45:36.313Z 14 280-299 88 88 88 151 151 151 F F F
96622024-02-20T22:45:36.317Z 15 300-319 81 81 81 142 142 142 F F F
96632024-02-20T22:45:36.318Z 16 320-339 81 81 81 142 142 142 F F F
96642024-02-20T22:45:36.318Z 17 340-359 89 89 89 154 154 154 F F F
96652024-02-20T22:45:36.318Z 18 360-379 89 89 89 154 154 154 F F F
96662024-02-20T22:45:36.318Z 19 380-399 88 88 88 151 151 151 F F F
96672024-02-20T22:45:36.318Z 20 400-419 88 88 88 153 153 153 F F F
96682024-02-20T22:45:36.318Z 21 420-439 88 88 88 153 153 153 F F F
96692024-02-20T22:45:36.318Z 22 440-459 84 84 84 148 148 148 F F F
96702024-02-20T22:45:36.318Z 23 460-479 88 88 88 153 153 153 F F F
96712024-02-20T22:45:36.318Z 24 480-499 86 86 86 150 150 150 F F F
96722024-02-20T22:45:36.318Z 25 500-519 85 85 85 149 149 149 F F F
96732024-02-20T22:45:36.318Z 26 520-539 85 85 85 149 149 149 F F F
96742024-02-20T22:45:36.319Z 27 540-559 81 81 81 142 142 142 F F F
96752024-02-20T22:45:36.319Z 28 560-579 82 82 82 144 144 144 F F F
96762024-02-20T22:45:36.319Z 29 580-599 88 88 88 151 151 151 F F F
96772024-02-20T22:45:36.319ZMax gen: 89, Max flush: 154
96782024-02-20T22:45:36.319Z
96792024-02-20T22:45:36.319ZBegin loop 45
96802024-02-20T22:45:36.319ZDownstairs to restart: 0
96812024-02-20T22:45:36.333Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.85677213Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498}
96822024-02-20T22:45:36.341Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857086859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498}
96832024-02-20T22:45:36.341Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857121408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498}
96842024-02-20T22:45:36.341Z{"msg":"Crucible 2ea28d12-3f42-4ff7-bcd6-1fadbe14f15d has session id: e67433f1-0990-42a3-9dc7-7609e2ed603c","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857136937ZCrucible runtime is spawned
96852024-02-20T22:45:36.341Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96862024-02-20T22:45:36.341Z{"msg":"Upstairs opts: Upstairs UUID: 2ea28d12-3f42-4ff7-bcd6-1fadbe14f15d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857172256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96872024-02-20T22:45:36.341Z{"msg":"Crucible stats registered with UUID: 2ea28d12-3f42-4ff7-bcd6-1fadbe14f15d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857206405Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96882024-02-20T22:45:36.341Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857222864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498}
96892024-02-20T22:45:36.341Z{"msg":"2ea28d12-3f42-4ff7-bcd6-1fadbe14f15d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.8573414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96902024-02-20T22:45:36.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857368599Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96912024-02-20T22:45:36.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857392058Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96922024-02-20T22:45:36.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857431477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96932024-02-20T22:45:36.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857468965Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96942024-02-20T22:45:36.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857586851Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96952024-02-20T22:45:36.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857680008Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96962024-02-20T22:45:36.342Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857718947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96972024-02-20T22:45:36.342Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857750986Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96982024-02-20T22:45:36.342Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857777794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
96992024-02-20T22:45:36.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857824573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97002024-02-20T22:45:36.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857851462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97012024-02-20T22:45:36.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.857884811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97022024-02-20T22:45:36.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.858431451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97032024-02-20T22:45:36.342Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.85847194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97042024-02-20T22:45:36.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.858497949Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97052024-02-20T22:45:36.342Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.858539377Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97062024-02-20T22:45:36.343Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.858569026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97072024-02-20T22:45:36.343Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.858598265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97082024-02-20T22:45:36.343Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859086808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97092024-02-20T22:45:36.343Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859124736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97102024-02-20T22:45:36.343Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.85930266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97112024-02-20T22:45:36.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859706005Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97122024-02-20T22:45:36.343Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859758764Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97132024-02-20T22:45:36.343Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859779493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97142024-02-20T22:45:36.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859823561Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97152024-02-20T22:45:36.343Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859840701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97162024-02-20T22:45:36.343Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.85986854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97172024-02-20T22:45:36.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859893799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97182024-02-20T22:45:36.343Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859908918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97192024-02-20T22:45:36.344Z{"msg":"[0]R flush_numbers[0..12]: [150, 151, 152, 136, 142, 151, 150, 146, 142, 147, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859929728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97202024-02-20T22:45:36.344Z{"msg":"[0]R generation[0..12]: [86, 88, 88, 77, 81, 88, 86, 84, 81, 84, 89, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859944427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97212024-02-20T22:45:36.344Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859965456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97222024-02-20T22:45:36.344Z{"msg":"[1]R flush_numbers[0..12]: [150, 151, 152, 136, 142, 151, 150, 146, 142, 147, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859980166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97232024-02-20T22:45:36.344Z{"msg":"[1]R generation[0..12]: [86, 88, 88, 77, 81, 88, 86, 84, 81, 84, 89, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.859994765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97242024-02-20T22:45:36.344Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860009535Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97252024-02-20T22:45:36.344Z{"msg":"[2]R flush_numbers[0..12]: [150, 151, 152, 136, 142, 151, 150, 146, 142, 147, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860024804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97262024-02-20T22:45:36.344Z{"msg":"[2]R generation[0..12]: [86, 88, 88, 77, 81, 88, 86, 84, 81, 84, 89, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860051113Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97272024-02-20T22:45:36.344Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860073032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603cWait for a query_work_queue command to finish before sending IO
97282024-02-20T22:45:36.344Z"}
97292024-02-20T22:45:36.344Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860117931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97302024-02-20T22:45:36.344Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860158469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97312024-02-20T22:45:36.344Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860174459Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97322024-02-20T22:45:36.344Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860188798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97332024-02-20T22:45:36.345Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860202908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97342024-02-20T22:45:36.345Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860215507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97352024-02-20T22:45:36.345Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860229027Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97362024-02-20T22:45:36.345Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860249046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97372024-02-20T22:45:36.345Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860286565Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97382024-02-20T22:45:36.345Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860310274Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97392024-02-20T22:45:36.345Z{"msg":"2ea28d12-3f42-4ff7-bcd6-1fadbe14f15d is now active with session: e67433f1-0990-42a3-9dc7-7609e2ed603c","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860332223Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97402024-02-20T22:45:36.345Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860346602Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498,"session_id":"e67433f1-0990-42a3-9dc7-7609e2ed603c"}
97412024-02-20T22:45:36.345Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:35.860369772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498}
97422024-02-20T22:45:36.345ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97432024-02-20T22:45:36.345ZLoading write count information from file "/tmp/test_repair_verify.data"
97442024-02-20T22:45:36.345ZRun Repair workload
97452024-02-20T22:45:36.345Z01/30 Write block 423 len 16384 data: 7 5 5 4
97462024-02-20T22:45:36.350Z02/30 Read block 152 len 28672
97472024-02-20T22:45:36.350Z03/30 Write block 32 len 28672 data: 15 14 11 11 10 9 10
97482024-02-20T22:45:36.350Z04/30 Read block 132 len 40960
97492024-02-20T22:45:36.350Z05/30 Read block 204 len 36864
97502024-02-20T22:45:36.350Z06/30 Write block 267 len 24576 data: 8 8 8 6 5 5
97512024-02-20T22:45:36.350Z07/30 Write block 480 len 4096 data: 13
97522024-02-20T22:45:36.350Z08/30 Read block 303 len 16384
97532024-02-20T22:45:36.350Z09/30 Read block 492 len 32768
97542024-02-20T22:45:36.350Z10/30 Write block 513 len 8192 data: 8 8
97552024-02-20T22:45:36.350Z11/30 Read block 530 len 28672
97562024-02-20T22:45:36.350Z12/30 Read block 403 len 4096
97572024-02-20T22:45:36.350Z13/30 Write block 437 len 20480 data: 5 7 9 11 11
97582024-02-20T22:45:36.350Z14/30 Flush
97592024-02-20T22:45:36.354Z15/30 Write block 284 len 8192 data: 9 9
97602024-02-20T22:45:36.354Z16/30 Write block 420 len 8192 data: 10 8
97612024-02-20T22:45:36.354Z17/30 Read block 206 len 8192
97622024-02-20T22:45:36.354Z18/30 Flush
97632024-02-20T22:45:36.358Z19/30 Write block 520 len 16384 data: 6 7 6 3
97642024-02-20T22:45:36.358Z20/30 Write block 67 len 32768 data: 7 8 9 8 8 7 5 6
97652024-02-20T22:45:36.358Z21/30 Read block 585 len 32768
97662024-02-20T22:45:36.358Z22/30 Write block 540 len 28672 data: 9 9 7 7 6 7 7
97672024-02-20T22:45:36.358Z23/30 Flush
97682024-02-20T22:45:36.362Z24/30 Write block 191 len 4096 data: 6
97692024-02-20T22:45:36.362Z25/30 Flush
97702024-02-20T22:45:36.366Z26/30 Write block 29 len 24576 data: 17 15 16 16 15 12
97712024-02-20T22:45:36.366Z27/30 Write block 333 len 40960 data: 10 10 8 9 9 8 9 9 10 11
97722024-02-20T22:45:36.366Z28/30 Read block 581 len 12288
97732024-02-20T22:45:36.366Z29/30 Read block 577 len 36864
97742024-02-20T22:45:36.395Z30/30 Write block 399 len 32768 data: 8 8 6 9 10 10 10 10
97752024-02-20T22:45:36.395Z----------------------------------------------------------------
97762024-02-20T22:45:36.395Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97772024-02-20T22:45:36.395ZStates: Active Active Active
97782024-02-20T22:45:36.395ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97792024-02-20T22:45:36.396Z 1 Acked 1000 Write 4 Sent Done Done false
97802024-02-20T22:45:36.396Z 2 Acked 1001 Read 7 Sent Done Done false
97812024-02-20T22:45:36.396Z 3 Acked 1002 Write 7 Sent Done Done false
97822024-02-20T22:45:36.396Z 4 Acked 1003 Read 10 Sent Done Done false
97832024-02-20T22:45:36.396Z 5 Acked 1004 Read 9 Sent Done Done false
97842024-02-20T22:45:36.396Z 6 Acked 1005 Write 6 Sent Done Done false
97852024-02-20T22:45:36.396Z 7 Acked 1006 Write 1 Sent Done Done false
97862024-02-20T22:45:36.396Z 8 Acked 1007 Read 4 Sent Done Done false
97872024-02-20T22:45:36.396Z 9 Acked 1008 Read 8 Sent Done Done false
97882024-02-20T22:45:36.396Z 10 Acked 1009 Write 2 Sent Done Done false
97892024-02-20T22:45:36.396Z 11 Acked 1010 Read 7 Sent Done Done false
97902024-02-20T22:45:36.396Z 12 Acked 1011 Read 1 Sent Done Done false
97912024-02-20T22:45:36.396Z 13 Acked 1012 Write 5 Sent Done Done false
97922024-02-20T22:45:36.397Z 14 Acked 1013 Flush 0 Sent Done Done false
97932024-02-20T22:45:36.397Z 15 Acked 1014 Write 2 Sent Done Done false
97942024-02-20T22:45:36.397Z 16 Acked 1015 Write 2 Sent Done Done false
97952024-02-20T22:45:36.397Z 17 Acked 1016 Read 2 Sent Done Done false
97962024-02-20T22:45:36.397Z 18 Acked 1017 Flush 0 Sent Done Done false
97972024-02-20T22:45:36.397Z 19 Acked 1018 Write 4 Sent Done Done false
97982024-02-20T22:45:36.397Z 20 Acked 1019 Write 8 Sent Done Done false
97992024-02-20T22:45:36.397Z 21 Acked 1020 Read 8 Sent Done Done false
98002024-02-20T22:45:36.397Z 22 Acked 1021 Write 7 Sent Done Done false
98012024-02-20T22:45:36.397Z 23 Acked 1022 Flush 0 Sent Done Done false
98022024-02-20T22:45:36.397Z 24 Acked 1023 Write 1 Sent Done Done false
98032024-02-20T22:45:36.397Z 25 Acked 1024 Flush 0 Sent Done Done false
98042024-02-20T22:45:36.397Z 26 Acked 1025 Write 6 Sent Done Done false
98052024-02-20T22:45:36.398Z 27 Acked 1026 Write 10 Sent Done Done false
98062024-02-20T22:45:36.398Z 28 Acked 1027 Read 3 Sent Done Done false
98072024-02-20T22:45:36.398Z 29 Acked 1028 Read 9 Sent Done Done false
98082024-02-20T22:45:36.398Z 30 Acked 1029 Write 8 Sent Sent Sent false
98092024-02-20T22:45:36.398Z STATES DS:0 DS:1 DS:2 TOTAL
98102024-02-20T22:45:36.398Z New 0 0 0 0
98112024-02-20T22:45:36.398Z Sent 30 1 1 32
98122024-02-20T22:45:36.398Z Done 0 29 29 58
98132024-02-20T22:45:36.398Z Skipped 0 0 0 0
98142024-02-20T22:45:36.398Z Error 0 0 0 0
98152024-02-20T22:45:36.398ZLast Flush: 0 1024 1024
98162024-02-20T22:45:36.398ZDownstairs last five completed:
98172024-02-20T22:45:36.398ZUpstairs last five completed: 30 29 28 27 26
98182024-02-20T22:45:36.398Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:35.892640187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1498}
98192024-02-20T22:45:36.399ZWrote out file "/tmp/test_repair_verify.data"
98202024-02-20T22:45:36.399Z
98212024-02-20T22:45:36.399ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98222024-02-20T22:45:36.399ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98232024-02-20T22:45:36.399Z 0 000-019 86 86 86 150 150 150 F F F
98242024-02-20T22:45:36.399Z 1 020-039 88 90 90 151 155 155 F T T <---
98252024-02-20T22:45:36.399Z 2 040-059 88 88 88 152 152 152 F F F
98262024-02-20T22:45:36.399Z 3 060-079 77 90 90 136 157 157 F F F <---
98272024-02-20T22:45:36.399Z 4 080-099 81 81 81 142 142 142 F F F
98282024-02-20T22:45:36.399Z 5 100-119 88 88 88 151 151 151 F F F
98292024-02-20T22:45:36.399Z 6 120-139 86 86 86 150 150 150 F F F
98302024-02-20T22:45:36.399Z 7 140-159 84 84 84 146 146 146 F F F
98312024-02-20T22:45:36.399Z 8 160-179 81 81 81 142 142 142 F F F
98322024-02-20T22:45:36.400Z 9 180-199 84 90 90 147 158 158 F F F <---
98332024-02-20T22:45:36.400Z 10 200-219 89 89 89 154 154 154 F F F
98342024-02-20T22:45:36.400Z 11 220-239 85 85 85 149 149 149 F F F
98352024-02-20T22:45:36.400Z 12 240-259 81 81 81 142 142 142 F F F
98362024-02-20T22:45:36.400Z 13 260-279 89 90 90 154 155 155 F F F <---
98372024-02-20T22:45:36.400Z 14 280-299 88 90 90 151 156 156 F F F <---
98382024-02-20T22:45:36.400Z 15 300-319 81 81 81 142 142 142 F F F
98392024-02-20T22:45:36.400Z 16 320-339 81 81 81 142 142 142 F T T <---
98402024-02-20T22:45:36.400Z 17 340-359 89 89 89 154 154 154 F T T <---
98412024-02-20T22:45:36.400Z 18 360-379 89 89 89 154 154 154 F F F
98422024-02-20T22:45:36.400Z 19 380-399 88 88 88 151 151 151 F T T <---
98432024-02-20T22:45:36.400Z 20 400-419 88 88 88 153 153 153 F T T <---
98442024-02-20T22:45:36.400Z 21 420-439 88 90 90 153 156 156 F F F <---
98452024-02-20T22:45:36.401Z 22 440-459 84 90 90 148 155 155 F F F <---
98462024-02-20T22:45:36.401Z 23 460-479 88 88 88 153 153 153 F F F
98472024-02-20T22:45:36.401Z 24 480-499 86 90 90 150 155 155 F F F <---
98482024-02-20T22:45:36.401Z 25 500-519 85 90 90 149 155 155 F F F <---
98492024-02-20T22:45:36.401Z 26 520-539 85 90 90 149 157 157 F F F <---
98502024-02-20T22:45:36.401Z 27 540-559 81 90 90 142 157 157 F F F <---
98512024-02-20T22:45:36.401Z 28 560-579 82 82 82 144 144 144 F F F
98522024-02-20T22:45:36.401Z 29 580-599 88 88 88 151 151 151 F F F
98532024-02-20T22:45:36.401ZMax gen: 90, Max flush: 158
98542024-02-20T22:45:36.401ZError: Difference in extent metadata found!
98552024-02-20T22:45:36.407ZA Difference in extent metadata is expected here
98562024-02-20T22:45:36.408ZOn loop 45
98572024-02-20T22:45:36.408Z
98582024-02-20T22:45:36.408ZVerifying data now
98592024-02-20T22:45:41.098ZNo change after verify
98602024-02-20T22:45:41.098ZLoop: 45 Downstairs dump after verify (and repair):
98612024-02-20T22:45:41.117ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98622024-02-20T22:45:41.117Z 0 000-019 86 86 86 150 150 150 F F F
98632024-02-20T22:45:41.117Z 1 020-039 91 91 91 159 159 159 F F F
98642024-02-20T22:45:41.117Z 2 040-059 88 88 88 152 152 152 F F F
98652024-02-20T22:45:41.117Z 3 060-079 90 90 90 157 157 157 F F F
98662024-02-20T22:45:41.117Z 4 080-099 81 81 81 142 142 142 F F F
98672024-02-20T22:45:41.118Z 5 100-119 88 88 88 151 151 151 F F F
98682024-02-20T22:45:41.118Z 6 120-139 86 86 86 150 150 150 F F F
98692024-02-20T22:45:41.118Z 7 140-159 84 84 84 146 146 146 F F F
98702024-02-20T22:45:41.118Z 8 160-179 81 81 81 142 142 142 F F F
98712024-02-20T22:45:41.118Z 9 180-199 90 90 90 158 158 158 F F F
98722024-02-20T22:45:41.129Z 10 200-219 89 89 89 154 154 154 F F F
98732024-02-20T22:45:41.129Z 11 220-239 85 85 85 149 149 149 F F F
98742024-02-20T22:45:41.129Z 12 240-259 81 81 81 142 142 142 F F F
98752024-02-20T22:45:41.129Z 13 260-279 90 90 90 155 155 155 F F F
98762024-02-20T22:45:41.129Z 14 280-299 90 90 90 156 156 156 F F F
98772024-02-20T22:45:41.129Z 15 300-319 81 81 81 142 142 142 F F F
98782024-02-20T22:45:41.129Z 16 320-339 91 91 91 159 159 159 F F F
98792024-02-20T22:45:41.129Z 17 340-359 91 91 91 159 159 159 F F F
98802024-02-20T22:45:41.129Z 18 360-379 89 89 89 154 154 154 F F F
98812024-02-20T22:45:41.129Z 19 380-399 91 91 91 159 159 159 F F F
98822024-02-20T22:45:41.129Z 20 400-419 91 91 91 159 159 159 F F F
98832024-02-20T22:45:41.129Z 21 420-439 90 90 90 156 156 156 F F F
98842024-02-20T22:45:41.130Z 22 440-459 90 90 90 155 155 155 F F F
98852024-02-20T22:45:41.130Z 23 460-479 88 88 88 153 153 153 F F F
98862024-02-20T22:45:41.130Z 24 480-499 90 90 90 155 155 155 F F F
98872024-02-20T22:45:41.130Z 25 500-519 90 90 90 155 155 155 F F F
98882024-02-20T22:45:41.130Z 26 520-539 90 90 90 157 157 157 F F F
98892024-02-20T22:45:41.130Z 27 540-559 90 90 90 157 157 157 F F F
98902024-02-20T22:45:41.130Z 28 560-579 82 82 82 144 144 144 F F F
98912024-02-20T22:45:41.130Z 29 580-599 88 88 88 151 151 151 F F F
98922024-02-20T22:45:41.130ZMax gen: 91, Max flush: 159
98932024-02-20T22:45:41.130Z
98942024-02-20T22:45:41.130ZBegin loop 46
98952024-02-20T22:45:41.130ZDownstairs to restart: 0
98962024-02-20T22:45:41.154Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.656966495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506}
98972024-02-20T22:45:41.154Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
98982024-02-20T22:45:41.154Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657352421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506}
98992024-02-20T22:45:41.154Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657405179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506}
99002024-02-20T22:45:41.155Z{"msg":"Crucible 8d91e115-4105-4e86-8564-4582dbae01f4 has session id: 6d6c75cf-8150-4694-b366-04c475754d2d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657430598Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99012024-02-20T22:45:41.155Z{"msg":"Upstairs opts: Upstairs UUID: 8d91e115-4105-4e86-8564-4582dbae01f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657475477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99022024-02-20T22:45:41.155Z{"msg":"Crucible stats registered with UUID: 8d91e115-4105-4e86-8564-4582dbae01f4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657493706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99032024-02-20T22:45:41.155Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657526905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506}
99042024-02-20T22:45:41.155Z{"msg":"8d91e115-4105-4e86-8564-4582dbae01f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657597422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99052024-02-20T22:45:41.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657638051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99062024-02-20T22:45:41.155Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.65765578Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99072024-02-20T22:45:41.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.65767046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99082024-02-20T22:45:41.155Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657684619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99092024-02-20T22:45:41.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657704769Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99102024-02-20T22:45:41.155Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657735768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99112024-02-20T22:45:41.155Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657756387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99122024-02-20T22:45:41.155Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657795865Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99132024-02-20T22:45:41.155Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657821734Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99142024-02-20T22:45:41.155Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.65793836Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99152024-02-20T22:45:41.155Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.657983549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99162024-02-20T22:45:41.155Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.658019257Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99172024-02-20T22:45:41.155Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.658701883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99182024-02-20T22:45:41.155Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.658741852Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99192024-02-20T22:45:41.155Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.65876856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99202024-02-20T22:45:41.155Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.658794859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99212024-02-20T22:45:41.155Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.658819619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99222024-02-20T22:45:41.155Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.658839718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99232024-02-20T22:45:41.155Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.659382629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99242024-02-20T22:45:41.155Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.659414288Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99252024-02-20T22:45:41.156Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.659429947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99262024-02-20T22:45:41.156Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.659868271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99272024-02-20T22:45:41.156Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.65991595Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99282024-02-20T22:45:41.156Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.659943109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99292024-02-20T22:45:41.156Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.659989217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99302024-02-20T22:45:41.156Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660020376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99312024-02-20T22:45:41.156Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660052925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99322024-02-20T22:45:41.156Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660085923Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99332024-02-20T22:45:41.156Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660118332Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99342024-02-20T22:45:41.156Z{"msg":"[0]R flush_numbers[0..12]: [150, 159, 152, 157, 142, 151, 150, 146, 142, 158, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660136942Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99352024-02-20T22:45:41.156Z{"msg":"[0]R generation[0..12]: [86, 91, 88, 90, 81, 88, 86, 84, 81, 90, 89, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.66016753Z","hostname":"Wait for a query_work_queue command to finish before sending IO
99362024-02-20T22:45:41.156Zip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99372024-02-20T22:45:41.156Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.66019204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99382024-02-20T22:45:41.156Z{"msg":"[1]R flush_numbers[0..12]: [150, 159, 152, 157, 142, 151, 150, 146, 142, 158, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660215679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99392024-02-20T22:45:41.156Z{"msg":"[1]R generation[0..12]: [86, 91, 88, 90, 81, 88, 86, 84, 81, 90, 89, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660245258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99402024-02-20T22:45:41.156Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660279777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99412024-02-20T22:45:41.156Z{"msg":"[2]R flush_numbers[0..12]: [150, 159, 152, 157, 142, 151, 150, 146, 142, 158, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660312425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99422024-02-20T22:45:41.156Z{"msg":"[2]R generation[0..12]: [86, 91, 88, 90, 81, 88, 86, 84, 81, 90, 89, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660347064Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99432024-02-20T22:45:41.156Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660381703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99442024-02-20T22:45:41.156Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660412312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99452024-02-20T22:45:41.156Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660436961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99462024-02-20T22:45:41.156Z{"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.66046331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99472024-02-20T22:45:41.156Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660486859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99482024-02-20T22:45:41.156Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660513938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99492024-02-20T22:45:41.156Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660535987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99502024-02-20T22:45:41.157ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99512024-02-20T22:45:41.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660566056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99522024-02-20T22:45:41.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660589935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99532024-02-20T22:45:41.157Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660616464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99542024-02-20T22:45:41.157Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660638484Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99552024-02-20T22:45:41.157Z{"msg":"8d91e115-4105-4e86-8564-4582dbae01f4 is now active with session: 6d6c75cf-8150-4694-b366-04c475754d2d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660666213Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99562024-02-20T22:45:41.157Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660687932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506,"session_id":"6d6c75cf-8150-4694-b366-04c475754d2d"}
99572024-02-20T22:45:41.157Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:40.660713871Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506}
99582024-02-20T22:45:41.165ZLoading write count information from file "/tmp/test_repair_verify.data"
99592024-02-20T22:45:41.165ZRun Repair workload
99602024-02-20T22:45:41.165Z01/30 Flush
99612024-02-20T22:45:41.165Z02/30 Write block 390 len 20480 data: 8 7 9 10 9
99622024-02-20T22:45:41.165Z03/30 Write block 207 len 24576 data: 10 8 8 8 8 7
99632024-02-20T22:45:41.165Z04/30 Read block 331 len 12288
99642024-02-20T22:45:41.165Z05/30 Write block 329 len 20480 data: 7 7 7 9 11
99652024-02-20T22:45:41.165Z06/30 Read block 215 len 36864
99662024-02-20T22:45:41.165Z07/30 Write block 142 len 24576 data: 7 9 8 7 5 4
99672024-02-20T22:45:41.165Z08/30 Flush
99682024-02-20T22:45:41.165Z09/30 Write block 396 len 8192 data: 11 13
99692024-02-20T22:45:41.165Z10/30 Read block 104 len 8192
99702024-02-20T22:45:41.165Z11/30 Read block 169 len 32768
99712024-02-20T22:45:41.165Z12/30 Write block 94 len 24576 data: 4 4 3 4 2 7
99722024-02-20T22:45:41.165Z13/30 Read block 7 len 28672
99732024-02-20T22:45:41.165Z14/30 Read block 256 len 40960
99742024-02-20T22:45:41.166Z15/30 Write block 338 len 8192 data: 9 10
99752024-02-20T22:45:41.166Z16/30 Write block 505 len 12288 data: 8 7 7
99762024-02-20T22:45:41.166Z17/30 Flush
99772024-02-20T22:45:41.166Z18/30 Write block 287 len 8192 data: 11 10
99782024-02-20T22:45:41.166Z19/30 Write block 245 len 20480 data: 3 3 7 6 5
99792024-02-20T22:45:41.166Z20/30 Write block 369 len 36864 data: 8 11 13 12 10 7 7 8 10
99802024-02-20T22:45:41.166Z21/30 Write block 115 len 8192 data: 8 9
99812024-02-20T22:45:41.166Z22/30 Write block 419 len 4096 data: 10
99822024-02-20T22:45:41.166Z23/30 Read block 124 len 36864
99832024-02-20T22:45:41.166Z24/30 Flush
99842024-02-20T22:45:41.166Z25/30 Write block 425 len 24576 data: 6 5 4 6 6 5
99852024-02-20T22:45:41.166Z26/30 Read block 127 len 28672
99862024-02-20T22:45:41.166Z27/30 Write block 573 len 16384 data: 4 4 5 7
99872024-02-20T22:45:41.166Z28/30 Write block 70 len 24576 data: 9 9 8 6 7 4
99882024-02-20T22:45:41.166Z29/30 Write block 423 len 16384 data: 8 6 7 6
99892024-02-20T22:45:41.172Z30/30 Write block 90 len 32768 data: 6 7 4 4 5 5 4 5
99902024-02-20T22:45:41.172Z----------------------------------------------------------------
99912024-02-20T22:45:41.172Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99922024-02-20T22:45:41.172ZStates: Active Active Active
99932024-02-20T22:45:41.172ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99942024-02-20T22:45:41.172Z 1 Acked 1000 Flush 0 Sent Done Done false
99952024-02-20T22:45:41.172Z 2 Acked 1001 Write 5 Sent Done Done false
99962024-02-20T22:45:41.172Z 3 Acked 1002 Write 6 Sent Done Done false
99972024-02-20T22:45:41.172Z 4 Acked 1003 Read 3 Sent Done Done false
99982024-02-20T22:45:41.172Z 5 Acked 1004 Write 5 Sent Done Done false
99992024-02-20T22:45:41.172Z 6 Acked 1005 Read 9 Sent Done Done false
100002024-02-20T22:45:41.172Z 7 Acked 1006 Write 6 Sent Done Done false
100012024-02-20T22:45:41.172Z 8 Acked 1007 Flush 0 Sent Done Done false
100022024-02-20T22:45:41.172Z 9 Acked 1008 Write 2 Sent Done Done false
100032024-02-20T22:45:41.172Z 10 Acked 1009 Read 2 Sent Done Done false
100042024-02-20T22:45:41.172Z 11 Acked 1010 Read 8 Sent Done Done false
100052024-02-20T22:45:41.172Z 12 Acked 1011 Write 6 Sent Done Done false
100062024-02-20T22:45:41.172Z 13 Acked 1012 Read 7 Sent Done Done false
100072024-02-20T22:45:41.172Z 14 Acked 1013 Read 10 Sent Done Done false
100082024-02-20T22:45:41.172Z 15 Acked 1014 Write 2 Sent Done Done false
100092024-02-20T22:45:41.172Z 16 Acked 1015 Write 3 Sent Done Done false
100102024-02-20T22:45:41.173Z 17 Acked 1016 Flush 0 Sent Done Done false
100112024-02-20T22:45:41.173Z 18 Acked 1017 Write 2 Sent Done Done false
100122024-02-20T22:45:41.173Z 19 Acked 1018 Write 5 Sent Done Done false
100132024-02-20T22:45:41.173Z 20 Acked 1019 Write 9 Sent Done Done false
100142024-02-20T22:45:41.173Z 21 Acked 1020 Write 2 Sent Done Done false
100152024-02-20T22:45:41.173Z 22 Acked 1021 Write 1 Sent Done Done false
100162024-02-20T22:45:41.173Z 23 Acked 1022 Read 9 Sent Done Done false
100172024-02-20T22:45:41.173Z 24 Acked 1023 Flush 0 Sent Done Done false
100182024-02-20T22:45:41.173Z 25 Acked 1024 Write 6 Sent Done Done false
100192024-02-20T22:45:41.173Z 26 Acked 1025 Read 7 Sent Done Done false
100202024-02-20T22:45:41.173Z 27 Acked 1026 Write 4 Sent Done Done false
100212024-02-20T22:45:41.173Z 28 Acked 1027 Write 6 Sent Done Done false
100222024-02-20T22:45:41.173Z 29 Acked 1028 Write 4 Sent Done Done false
100232024-02-20T22:45:41.173Z 30 Acked 1029 Write 8 Sent Sent Sent false
100242024-02-20T22:45:41.173Z STATES DS:0 DS:1 DS:2 TOTAL
100252024-02-20T22:45:41.173Z New 0 0 0 0
100262024-02-20T22:45:41.173Z Sent 30 1 1 32
100272024-02-20T22:45:41.173Z Done 0 29 29 58
100282024-02-20T22:45:41.173Z Skipped 0 0 0 0
100292024-02-20T22:45:41.173Z Error 0 0 0 0
100302024-02-20T22:45:41.173ZLast Flush: 0 1023 1023
100312024-02-20T22:45:41.173ZDownstairs last five completed:
100322024-02-20T22:45:41.173ZUpstairs last five completed: 30 29 28 27 26
100332024-02-20T22:45:41.173Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:40.689481592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1506}
100342024-02-20T22:45:41.173ZWrote out file "/tmp/test_repair_verify.data"
100352024-02-20T22:45:41.174Z
100362024-02-20T22:45:41.183ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100372024-02-20T22:45:41.207ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100382024-02-20T22:45:41.207Z 0 000-019 86 86 86 150 150 150 F F F
100392024-02-20T22:45:41.207Z 1 020-039 91 91 91 159 159 159 F F F
100402024-02-20T22:45:41.207Z 2 040-059 88 88 88 152 152 152 F F F
100412024-02-20T22:45:41.207Z 3 060-079 90 90 90 157 157 157 F T T <---
100422024-02-20T22:45:41.207Z 4 080-099 81 92 92 142 162 162 F T T <---
100432024-02-20T22:45:41.207Z 5 100-119 88 92 92 151 163 163 F F F <---
100442024-02-20T22:45:41.207Z 6 120-139 86 86 86 150 150 150 F F F
100452024-02-20T22:45:41.207Z 7 140-159 84 92 92 146 161 161 F F F <---
100462024-02-20T22:45:41.207Z 8 160-179 81 81 81 142 142 142 F F F
100472024-02-20T22:45:41.207Z 9 180-199 90 90 90 158 158 158 F F F
100482024-02-20T22:45:41.207Z 10 200-219 89 92 92 154 161 161 F F F <---
100492024-02-20T22:45:41.207Z 11 220-239 85 85 85 149 149 149 F F F
100502024-02-20T22:45:41.207Z 12 240-259 81 92 92 142 163 163 F F F <---
100512024-02-20T22:45:41.207Z 13 260-279 90 90 90 155 155 155 F F F
100522024-02-20T22:45:41.207Z 14 280-299 90 92 92 156 163 163 F F F <---
100532024-02-20T22:45:41.207Z 15 300-319 81 81 81 142 142 142 F F F
100542024-02-20T22:45:41.207Z 16 320-339 91 92 92 159 162 162 F F F <---
100552024-02-20T22:45:41.208Z 17 340-359 91 91 91 159 159 159 F F F
100562024-02-20T22:45:41.208Z 18 360-379 89 92 92 154 163 163 F F F <---
100572024-02-20T22:45:41.208Z 19 380-399 91 92 92 159 162 162 F F F <---
100582024-02-20T22:45:41.208Z 20 400-419 91 92 92 159 163 163 F F F <---
100592024-02-20T22:45:41.208Z 21 420-439 90 90 90 156 156 156 F T T <---
100602024-02-20T22:45:41.208Z 22 440-459 90 90 90 155 155 155 F F F
100612024-02-20T22:45:41.208Z 23 460-479 88 88 88 153 153 153 F F F
100622024-02-20T22:45:41.208Z 24 480-499 90 90 90 155 155 155 F F F
100632024-02-20T22:45:41.208Z 25 500-519 90 92 92 155 162 162 F F F <---
100642024-02-20T22:45:41.208Z 26 520-539 90 90 90 157 157 157 F F F
100652024-02-20T22:45:41.208Z 27 540-559 90 90 90 157 157 157 F F F
100662024-02-20T22:45:41.208Z 28 560-579 82 82 82 144 144 144 F T T <---
100672024-02-20T22:45:41.208Z 29 580-599 88 88 88 151 151 151 F F F
100682024-02-20T22:45:41.208ZMax gen: 92, Max flush: 163
100692024-02-20T22:45:41.208ZError: Difference in extent metadata found!
100702024-02-20T22:45:41.208ZA Difference in extent metadata is expected here
100712024-02-20T22:45:41.208ZOn loop 46
100722024-02-20T22:45:41.208Z
100732024-02-20T22:45:41.208ZVerifying data now
100742024-02-20T22:45:45.853ZNo change after verify
100752024-02-20T22:45:45.853ZLoop: 46 Downstairs dump after verify (and repair):
100762024-02-20T22:45:45.867ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100772024-02-20T22:45:45.867Z 0 000-019 86 86 86 150 150 150 F F F
100782024-02-20T22:45:45.867Z 1 020-039 91 91 91 159 159 159 F F F
100792024-02-20T22:45:45.867Z 2 040-059 88 88 88 152 152 152 F F F
100802024-02-20T22:45:45.867Z 3 060-079 93 93 93 164 164 164 F F F
100812024-02-20T22:45:45.867Z 4 080-099 93 93 93 164 164 164 F F F
100822024-02-20T22:45:45.867Z 5 100-119 92 92 92 163 163 163 F F F
100832024-02-20T22:45:45.867Z 6 120-139 86 86 86 150 150 150 F F F
100842024-02-20T22:45:45.867Z 7 140-159 92 92 92 161 161 161 F F F
100852024-02-20T22:45:45.867Z 8 160-179 81 81 81 142 142 142 F F F
100862024-02-20T22:45:45.867Z 9 180-199 90 90 90 158 158 158 F F F
100872024-02-20T22:45:45.872Z 10 200-219 92 92 92 161 161 161 F F F
100882024-02-20T22:45:45.872Z 11 220-239 85 85 85 149 149 149 F F F
100892024-02-20T22:45:45.872Z 12 240-259 92 92 92 163 163 163 F F F
100902024-02-20T22:45:45.872Z 13 260-279 90 90 90 155 155 155 F F F
100912024-02-20T22:45:45.872Z 14 280-299 92 92 92 163 163 163 F F F
100922024-02-20T22:45:45.872Z 15 300-319 81 81 81 142 142 142 F F F
100932024-02-20T22:45:45.872Z 16 320-339 92 92 92 162 162 162 F F F
100942024-02-20T22:45:45.872Z 17 340-359 91 91 91 159 159 159 F F F
100952024-02-20T22:45:45.872Z 18 360-379 92 92 92 163 163 163 F F F
100962024-02-20T22:45:45.872Z 19 380-399 92 92 92 162 162 162 F F F
100972024-02-20T22:45:45.872Z 20 400-419 92 92 92 163 163 163 F F F
100982024-02-20T22:45:45.872Z 21 420-439 93 93 93 164 164 164 F F F
100992024-02-20T22:45:45.872Z 22 440-459 90 90 90 155 155 155 F F F
101002024-02-20T22:45:45.872Z 23 460-479 88 88 88 153 153 153 F F F
101012024-02-20T22:45:45.872Z 24 480-499 90 90 90 155 155 155 F F F
101022024-02-20T22:45:45.872Z 25 500-519 92 92 92 162 162 162 F F F
101032024-02-20T22:45:45.873Z 26 520-539 90 90 90 157 157 157 F F F
101042024-02-20T22:45:45.873Z 27 540-559 90 90 90 157 157 157 F F F
101052024-02-20T22:45:45.873Z 28 560-579 93 93 93 164 164 164 F F F
101062024-02-20T22:45:45.873Z 29 580-599 88 88 88 151 151 151 F F F
101072024-02-20T22:45:45.873ZMax gen: 93, Max flush: 164
101082024-02-20T22:45:45.873Z
101092024-02-20T22:45:45.873ZBegin loop 47
101102024-02-20T22:45:45.873ZDownstairs to restart: 0
101112024-02-20T22:45:45.887Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.410493351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514}
101122024-02-20T22:45:45.887Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.410767721Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514}
101132024-02-20T22:45:45.896Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.41081163Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514}
101142024-02-20T22:45:45.896Z{"msg":"Crucible d1daa202-271e-4fde-aa23-6266f15a0b37 has session id: 1f46e675-7272-4852-9710-e4909dfcc10f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.410849658Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101152024-02-20T22:45:45.896Z{"msg":"Upstairs opts: Upstairs UUID: d1daa202-271e-4fde-aa23-6266f15a0b37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,Crucible runtime is spawned
101162024-02-20T22:45:45.896Z"time":"2024-02-20T22:45:45.410920955Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101172024-02-20T22:45:45.896Z{"msg":"Crucible stats registered with UUID: d1daa202-271e-4fde-aa23-6266f15a0b37","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.410965864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101182024-02-20T22:45:45.896Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411006783Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514}
101192024-02-20T22:45:45.896Z{"msg":"d1daa202-271e-4fde-aa23-6266f15a0b37 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411120519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101202024-02-20T22:45:45.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411160437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101212024-02-20T22:45:45.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411215075Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101222024-02-20T22:45:45.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411247344Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101232024-02-20T22:45:45.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411294192Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101242024-02-20T22:45:45.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411338341Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101252024-02-20T22:45:45.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411388369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101262024-02-20T22:45:45.896Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411425097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101272024-02-20T22:45:45.897Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411556733Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101282024-02-20T22:45:45.897Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.41164328Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101292024-02-20T22:45:45.897Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411685508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101302024-02-20T22:45:45.897Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411712177Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101312024-02-20T22:45:45.897Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.411737106Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101322024-02-20T22:45:45.897Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412237648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101332024-02-20T22:45:45.897Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412274747Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101342024-02-20T22:45:45.897Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412294897Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101352024-02-20T22:45:45.897Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412321785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101362024-02-20T22:45:45.897Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412339255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101372024-02-20T22:45:45.897Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412393503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101382024-02-20T22:45:45.897Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412826127Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101392024-02-20T22:45:45.897Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.412871716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101402024-02-20T22:45:45.897Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413059919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101412024-02-20T22:45:45.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413483624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101422024-02-20T22:45:45.897Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413541002Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101432024-02-20T22:45:45.897Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.41359087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101442024-02-20T22:45:45.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413624869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101452024-02-20T22:45:45.897Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413660217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101462024-02-20T22:45:45.897Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413683567Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101472024-02-20T22:45:45.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413715595Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101482024-02-20T22:45:45.897Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413750614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101492024-02-20T22:45:45.897ZWait for a query_work_queue command to finish before sending IO
101502024-02-20T22:45:45.897Z{"msg":"[0]R flush_numbers[0..12]: [150, 159, 152, 164, 164, 163, 150, 161, 142, 158, 161, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413795503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101512024-02-20T22:45:45.898Z{"msg":"[0]R generation[0..12]: [86, 91, 88, 93, 93, 92, 86, 92, 81, 90, 92, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413835891Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101522024-02-20T22:45:45.898Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.41387586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101532024-02-20T22:45:45.898Z{"msg":"[1]R flush_numbers[0..12]: [150, 159, 152, 164, 164, 163, 150, 161, 142, 158, 161, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413907129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101542024-02-20T22:45:45.898Z{"msg":"[1]R generation[0..12]: [86, 91, 88, 93, 93, 92, 86, 92, 81, 90, 92, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413946157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101552024-02-20T22:45:45.898Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.413985826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101562024-02-20T22:45:45.898Z{"msg":"[2]R flush_numbers[0..12]: [150, 159, 152, 164, 164, 163, 150, 161, 142, 158, 161, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414026115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101572024-02-20T22:45:45.898Z{"msg":"[2]R generation[0..12]: [86, 91, 88, 93, 93, 92, 86, 92, 81, 90, 92, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414065013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101582024-02-20T22:45:45.898Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414104562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101592024-02-20T22:45:45.898Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.41414094Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101602024-02-20T22:45:45.898Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414177169Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101612024-02-20T22:45:45.898Zip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101622024-02-20T22:45:45.898Z{"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414213798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101632024-02-20T22:45:45.898Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414238527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101642024-02-20T22:45:45.898Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414252906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101652024-02-20T22:45:45.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414266206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101662024-02-20T22:45:45.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414286515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101672024-02-20T22:45:45.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414299915Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101682024-02-20T22:45:45.898Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414319534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101692024-02-20T22:45:45.898Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414332073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101702024-02-20T22:45:45.898Z{"msg":"d1daa202-271e-4fde-aa23-6266f15a0b37 is now active with session: 1f46e675-7272-4852-9710-e4909dfcc10f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414345433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101712024-02-20T22:45:45.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414363972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514,"session_id":"1f46e675-7272-4852-9710-e4909dfcc10f"}
101722024-02-20T22:45:45.898Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:45.414376672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514}
101732024-02-20T22:45:45.898ZLoading write count information from file "/tmp/test_repair_verify.data"
101742024-02-20T22:45:45.898ZRun Repair workload
101752024-02-20T22:45:45.898Z01/30 Flush
101762024-02-20T22:45:45.904Z02/30 Write block 369 len 36864 data: 9 12 14 13 11 8 8 9 11
101772024-02-20T22:45:45.904Z03/30 Flush
101782024-02-20T22:45:45.904Z04/30 Write block 174 len 32768 data: 8 10 9 9 9 10 10 10
101792024-02-20T22:45:45.904Z05/30 Write block 531 len 40960 data: 9 6 7 9 10 11 9 10 8 10
101802024-02-20T22:45:45.904Z06/30 Write block 436 len 24576 data: 4 6 8 10 12 12
101812024-02-20T22:45:45.904Z07/30 Read block 97 len 24576
101822024-02-20T22:45:45.904Z08/30 Read block 551 len 28672
101832024-02-20T22:45:45.904Z09/30 Read block 492 len 20480
101842024-02-20T22:45:45.904Z10/30 Write block 187 len 8192 data: 7 7
101852024-02-20T22:45:45.904Z11/30 Read block 202 len 28672
101862024-02-20T22:45:45.909Z12/30 Write block 311 len 28672 data: 9 8 7 9 10 10 11
101872024-02-20T22:45:45.909Z13/30 Read block 193 len 12288
101882024-02-20T22:45:45.909Z14/30 Write block 119 len 24576 data: 8 9 8 8 5 5
101892024-02-20T22:45:45.909Z15/30 Read block 288 len 20480
101902024-02-20T22:45:45.910Z16/30 Write block 477 len 40960 data: 14 14 11 14 13 12 12 10 8 6
101912024-02-20T22:45:45.910Z17/30 Read block 408 len 32768
101922024-02-20T22:45:45.916Z18/30 Read block 13 len 36864
101932024-02-20T22:45:45.916Z19/30 Read block 71 len 24576
101942024-02-20T22:45:45.916Z20/30 Write block 235 len 8192 data: 12 12
101952024-02-20T22:45:45.916Z21/30 Flush
101962024-02-20T22:45:45.920Z22/30 Write block 253 len 24576 data: 6 5 3 3 4 6
101972024-02-20T22:45:45.920Z23/30 Read block 254 len 4096
101982024-02-20T22:45:45.920Z24/30 Read block 290 len 20480
101992024-02-20T22:45:45.920Z25/30 Write block 141 len 12288 data: 8 8 10
102002024-02-20T22:45:45.920Z26/30 Read block 498 len 4096
102012024-02-20T22:45:45.920Z27/30 Read block 43 len 16384
102022024-02-20T22:45:45.926Z28/30 Write block 307 len 16384 data: 10 9 11 9
102032024-02-20T22:45:45.926Z29/30 Read block 358 len 16384
102042024-02-20T22:45:45.926Z30/30 Read block 117 len 8192
102052024-02-20T22:45:45.926Z----------------------------------------------------------------
102062024-02-20T22:45:45.926Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102072024-02-20T22:45:45.926ZStates: Active Active Active
102082024-02-20T22:45:45.926ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102092024-02-20T22:45:45.926Z 1 Acked 1000 Flush 0 Sent Done Done false
102102024-02-20T22:45:45.926Z 2 Acked 1001 Write 9 Sent Done Done false
102112024-02-20T22:45:45.926Z 3 Acked 1002 Flush 0 Sent Done Done false
102122024-02-20T22:45:45.927Z 4 Acked 1003 Write 8 Sent Done Done false
102132024-02-20T22:45:45.927Z 5 Acked 1004 Write 10 Sent Done Done false
102142024-02-20T22:45:45.927Z 6 Acked 1005 Write 6 Sent Done Done false
102152024-02-20T22:45:45.927Z 7 Acked 1006 Read 6 Sent Done Done false
102162024-02-20T22:45:45.927Z 8 Acked 1007 Read 7 Sent Done Done false
102172024-02-20T22:45:45.927Z 9 Acked 1008 Read 5 Sent Done Done false
102182024-02-20T22:45:45.927Z 10 Acked 1009 Write 2 Sent Done Done false
102192024-02-20T22:45:45.927Z 11 Acked 1010 Read 7 Sent Done Done false
102202024-02-20T22:45:45.927Z 12 Acked 1011 Write 7 Sent Done Done false
102212024-02-20T22:45:45.927Z 13 Acked 1012 Read 3 Sent Done Done false
102222024-02-20T22:45:45.927Z 14 Acked 1013 Write 6 Sent Done Done false
102232024-02-20T22:45:45.927Z 15 Acked 1014 Read 5 Sent Done Done false
102242024-02-20T22:45:45.927Z 16 Acked 1015 Write 10 Sent Done Done false
102252024-02-20T22:45:45.927Z 17 Acked 1016 Read 8 Sent Done Done false
102262024-02-20T22:45:45.927Z 18 Acked 1017 Read 9 Sent Done Done false
102272024-02-20T22:45:45.927Z 19 Acked 1018 Read 6 Sent Done Done false
102282024-02-20T22:45:45.927Z 20 Acked 1019 Write 2 Sent Done Done false
102292024-02-20T22:45:45.927Z 21 Acked 1020 Flush 0 Sent Done Done false
102302024-02-20T22:45:45.927Z 22 Acked 1021 Write 6 Sent Done Done false
102312024-02-20T22:45:45.927Z 23 Acked 1022 Read 1 Sent Done Done false
102322024-02-20T22:45:45.927Z 24 Acked 1023 Read 5 Sent Done Done false
102332024-02-20T22:45:45.927Z 25 Acked 1024 Write 3 Sent Done Done false
102342024-02-20T22:45:45.927Z 26 Acked 1025 Read 1 Sent Done Done false
102352024-02-20T22:45:45.927Z 27 Acked 1026 Read 4 Sent Done Done false
102362024-02-20T22:45:45.927Z 28 Acked 1027 Write 4 Sent Done Done false
102372024-02-20T22:45:45.928Z 29 Acked 1028 Read 4 Sent Done Done false
102382024-02-20T22:45:45.928Z 30 Acked 1029 Read 2 Sent Done Done false
102392024-02-20T22:45:45.928Z STATES DS:0 DS:1 DS:2 TOTAL
102402024-02-20T22:45:45.928Z New 0 0 0 0
102412024-02-20T22:45:45.928Z Sent 30 0 0 30
102422024-02-20T22:45:45.928Z Done 0 30 30 60
102432024-02-20T22:45:45.928Z Skipped 0 0 0 0
102442024-02-20T22:45:45.928Z Error 0 0 0 0
102452024-02-20T22:45:45.928ZLast Flush: 0 1020 1020
102462024-02-20T22:45:45.928ZDownstairs last five completed:
102472024-02-20T22:45:45.928ZUpstairs last five completed: 30 29 28 27 26
102482024-02-20T22:45:45.928Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:45.446096487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1514}
102492024-02-20T22:45:45.928ZWrote out file "/tmp/test_repair_verify.data"
102502024-02-20T22:45:45.932Z
102512024-02-20T22:45:45.932ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102522024-02-20T22:45:45.945ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102532024-02-20T22:45:45.945Z 0 000-019 86 86 86 150 150 150 F F F
102542024-02-20T22:45:45.945Z 1 020-039 91 91 91 159 159 159 F F F
102552024-02-20T22:45:45.945Z 2 040-059 88 88 88 152 152 152 F F F
102562024-02-20T22:45:45.945Z 3 060-079 93 93 93 164 164 164 F F F
102572024-02-20T22:45:45.945Z 4 080-099 93 93 93 164 164 164 F F F
102582024-02-20T22:45:45.946Z 5 100-119 92 94 94 163 167 167 F F F <---
102592024-02-20T22:45:45.946Z 6 120-139 86 94 94 150 167 167 F F F <---
102602024-02-20T22:45:45.946Z 7 140-159 92 92 92 161 161 161 F T T <---
102612024-02-20T22:45:45.946Z 8 160-179 81 94 94 142 167 167 F F F <---
102622024-02-20T22:45:45.946Z 9 180-199 90 94 94 158 167 167 F F F <---
102632024-02-20T22:45:45.950Z 10 200-219 92 92 92 161 161 161 F F F
102642024-02-20T22:45:45.950Z 11 220-239 85 94 94 149 167 167 F F F <---
102652024-02-20T22:45:45.950Z 12 240-259 92 92 92 163 163 163 F T T <---
102662024-02-20T22:45:45.950Z 13 260-279 90 90 90 155 155 155 F F F
102672024-02-20T22:45:45.950Z 14 280-299 92 92 92 163 163 163 F F F
102682024-02-20T22:45:45.951Z 15 300-319 81 94 94 142 167 167 F T T <---
102692024-02-20T22:45:45.951Z 16 320-339 92 92 92 162 162 162 F F F
102702024-02-20T22:45:45.951Z 17 340-359 91 91 91 159 159 159 F F F
102712024-02-20T22:45:45.951Z 18 360-379 92 94 94 163 166 166 F F F <---
102722024-02-20T22:45:45.951Z 19 380-399 92 92 92 162 162 162 F F F
102732024-02-20T22:45:45.951Z 20 400-419 92 92 92 163 163 163 F F F
102742024-02-20T22:45:45.951Z 21 420-439 93 94 94 164 167 167 F F F <---
102752024-02-20T22:45:45.951Z 22 440-459 90 94 94 155 167 167 F F F <---
102762024-02-20T22:45:45.951Z 23 460-479 88 94 94 153 167 167 F F F <---
102772024-02-20T22:45:45.951Z 24 480-499 90 94 94 155 167 167 F F F <---
102782024-02-20T22:45:45.951Z 25 500-519 92 92 92 162 162 162 F F F
102792024-02-20T22:45:45.951Z 26 520-539 90 94 94 157 167 167 F F F <---
102802024-02-20T22:45:45.951Z 27 540-559 90 94 94 157 167 167 F F F <---
102812024-02-20T22:45:45.951Z 28 560-579 93 93 93 164 164 164 F F F
102822024-02-20T22:45:45.951Z 29 580-599 88 88 88 151 151 151 F F F
102832024-02-20T22:45:45.951ZMax gen: 94, Max flush: 167
102842024-02-20T22:45:45.951ZError: Difference in extent metadata found!
102852024-02-20T22:45:45.951ZA Difference in extent metadata is expected here
102862024-02-20T22:45:45.951ZOn loop 47
102872024-02-20T22:45:45.951Z
102882024-02-20T22:45:45.955ZVerifying data now
102892024-02-20T22:45:50.662ZNo change after verify
102902024-02-20T22:45:50.662ZLoop: 47 Downstairs dump after verify (and repair):
102912024-02-20T22:45:50.676ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102922024-02-20T22:45:50.676Z 0 000-019 86 86 86 150 150 150 F F F
102932024-02-20T22:45:50.676Z 1 020-039 91 91 91 159 159 159 F F F
102942024-02-20T22:45:50.676Z 2 040-059 88 88 88 152 152 152 F F F
102952024-02-20T22:45:50.676Z 3 060-079 93 93 93 164 164 164 F F F
102962024-02-20T22:45:50.676Z 4 080-099 93 93 93 164 164 164 F F F
102972024-02-20T22:45:50.676Z 5 100-119 94 94 94 167 167 167 F F F
102982024-02-20T22:45:50.676Z 6 120-139 94 94 94 167 167 167 F F F
102992024-02-20T22:45:50.676Z 7 140-159 95 95 95 168 168 168 F F F
103002024-02-20T22:45:50.676Z 8 160-179 94 94 94 167 167 167 F F F
103012024-02-20T22:45:50.676Z 9 180-199 94 94 94 167 167 167 F F F
103022024-02-20T22:45:50.677Z 10 200-219 92 92 92 161 161 161 F F F
103032024-02-20T22:45:50.677Z 11 220-239 94 94 94 167 167 167 F F F
103042024-02-20T22:45:50.677Z 12 240-259 95 95 95 168 168 168 F F F
103052024-02-20T22:45:50.677Z 13 260-279 90 90 90 155 155 155 F F F
103062024-02-20T22:45:50.684Z 14 280-299 92 92 92 163 163 163 F F F
103072024-02-20T22:45:50.684Z 15 300-319 95 95 95 168 168 168 F F F
103082024-02-20T22:45:50.684Z 16 320-339 92 92 92 162 162 162 F F F
103092024-02-20T22:45:50.684Z 17 340-359 91 91 91 159 159 159 F F F
103102024-02-20T22:45:50.684Z 18 360-379 94 94 94 166 166 166 F F F
103112024-02-20T22:45:50.684Z 19 380-399 92 92 92 162 162 162 F F F
103122024-02-20T22:45:50.684Z 20 400-419 92 92 92 163 163 163 F F F
103132024-02-20T22:45:50.684Z 21 420-439 94 94 94 167 167 167 F F F
103142024-02-20T22:45:50.684Z 22 440-459 94 94 94 167 167 167 F F F
103152024-02-20T22:45:50.684Z 23 460-479 94 94 94 167 167 167 F F F
103162024-02-20T22:45:50.684Z 24 480-499 94 94 94 167 167 167 F F F
103172024-02-20T22:45:50.684Z 25 500-519 92 92 92 162 162 162 F F F
103182024-02-20T22:45:50.684Z 26 520-539 94 94 94 167 167 167 F F F
103192024-02-20T22:45:50.684Z 27 540-559 94 94 94 167 167 167 F F F
103202024-02-20T22:45:50.685Z 28 560-579 93 93 93 164 164 164 F F F
103212024-02-20T22:45:50.685Z 29 580-599 88 88 88 151 151 151 F F F
103222024-02-20T22:45:50.685ZMax gen: 95, Max flush: 168
103232024-02-20T22:45:50.685Z
103242024-02-20T22:45:50.685ZBegin loop 48
103252024-02-20T22:45:50.685ZDownstairs to restart: 0
103262024-02-20T22:45:50.700Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220228434Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522}
103272024-02-20T22:45:50.709Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220545132Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522}
103282024-02-20T22:45:50.709Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
103292024-02-20T22:45:50.709Z,"time":"2024-02-20T22:45:50.220593851Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522}
103302024-02-20T22:45:50.709Z{"msg":"Crucible 2503d529-e350-4925-8ab3-c8743aef4abe has session id: 81a4bee4-ceab-4173-a59e-313226d9b47f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220637829Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103312024-02-20T22:45:50.709Z{"msg":"Upstairs opts: Upstairs UUID: 2503d529-e350-4925-8ab3-c8743aef4abe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220686218Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103322024-02-20T22:45:50.709Z{"msg":"Crucible stats registered with UUID: 2503d529-e350-4925-8ab3-c8743aef4abe","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220712646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103332024-02-20T22:45:50.709Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220760805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522}
103342024-02-20T22:45:50.709Z{"msg":"2503d529-e350-4925-8ab3-c8743aef4abe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220845182Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103352024-02-20T22:45:50.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.22088738Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103362024-02-20T22:45:50.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220933969Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103372024-02-20T22:45:50.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220963097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103382024-02-20T22:45:50.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.220989437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103392024-02-20T22:45:50.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221015346Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103402024-02-20T22:45:50.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221040725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103412024-02-20T22:45:50.709Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221066494Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103422024-02-20T22:45:50.709Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221114272Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103432024-02-20T22:45:50.709Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221149281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103442024-02-20T22:45:50.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221185449Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103452024-02-20T22:45:50.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221209219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103462024-02-20T22:45:50.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221243927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103472024-02-20T22:45:50.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221934283Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103482024-02-20T22:45:50.710Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.221981431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103492024-02-20T22:45:50.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222045339Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103502024-02-20T22:45:50.710Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222089177Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103512024-02-20T22:45:50.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222137786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103522024-02-20T22:45:50.710Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222165765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103532024-02-20T22:45:50.710Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222633858Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103542024-02-20T22:45:50.710Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222684266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103552024-02-20T22:45:50.710Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.222731044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103562024-02-20T22:45:50.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223232766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103572024-02-20T22:45:50.710Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223284465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103582024-02-20T22:45:50.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223307744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103592024-02-20T22:45:50.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223333983Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103602024-02-20T22:45:50.710Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223366482Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103612024-02-20T22:45:50.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223391291Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103622024-02-20T22:45:50.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.22340543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103632024-02-20T22:45:50.710Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.22341994Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103642024-02-20T22:45:50.710Z{"msg":"[0]R flush_numbers[0..12]: [150, 159, 152, 164, 164, 167, 167, 168, 167, 167, 161, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223433979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103652024-02-20T22:45:50.710Z{"msg":"[0]R generation[0..12]: [86, 91, 88, 93, 93, 94, 94, 95, 94, 94, 92, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223448569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103662024-02-20T22:45:50.710Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223463278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103672024-02-20T22:45:50.710Z{"msg":"[1]R flush_numbers[0..12]: [150, 159, 152, 164, 164, 167, 167, 168, 167, 167, 161, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223478067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103682024-02-20T22:45:50.710Z{"msg"Wait for a query_work_queue command to finish before sending IO
103692024-02-20T22:45:50.710Z:"[1]R generation[0..12]: [86, 91, 88, 93, 93, 94, 94, 95, 94, 94, 92, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223495717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103702024-02-20T22:45:50.710Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223529106Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103712024-02-20T22:45:50.711Z{"msg":"[2]R flush_numbers[0..12]: [150, 159, 152, 164, 164, 167, 167, 168, 167, 167, 161, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223549415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103722024-02-20T22:45:50.711Z{"msg":"[2]R generation[0..12]: [86, 91, 88, 93, 93, 94, 94, 95, 94, 94, 92, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223575394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103732024-02-20T22:45:50.711Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223600353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103742024-02-20T22:45:50.711Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223642022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103752024-02-20T22:45:50.711Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.22368026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103762024-02-20T22:45:50.711Z{"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223713389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103772024-02-20T22:45:50.711Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223740758Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103782024-02-20T22:45:50.711Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223778337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103792024-02-20T22:45:50.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223812575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103802024-02-20T22:45:50.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223849524Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103812024-02-20T22:45:50.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223887403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103822024-02-20T22:45:50.711Z,"pid":1522,"client":"2","":"downstairs","session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103832024-02-20T22:45:50.711Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223926222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103842024-02-20T22:45:50.711Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.223941841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103852024-02-20T22:45:50.711Z{"msg":"2503d529-e350-4925-8ab3-c8743aef4abe is now active with session: 81a4bee4-ceab-4173-a59e-313226d9b47f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.22395489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103862024-02-20T22:45:50.711Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.2239806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522,"session_id":"81a4bee4-ceab-4173-a59e-313226d9b47f"}
103872024-02-20T22:45:50.711Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:50.224005109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522}
103882024-02-20T22:45:50.711ZLoading write count information from file "/tmp/test_repair_verify.data"
103892024-02-20T22:45:50.711ZRun Repair workload
103902024-02-20T22:45:50.711Z01/30 Write block 2 len 8192 data: 6 7
103912024-02-20T22:45:50.711Z02/30 Write block 108 len 20480 data: 6 7 9 10 10
103922024-02-20T22:45:50.711Z03/30 Read block 85 len 20480
103932024-02-20T22:45:50.717Z04/30 Flush
103942024-02-20T22:45:50.717Z05/30 Write block 411 len 20480 data: 11 11 11 11 11
103952024-02-20T22:45:50.717Z06/30 Write block 332 len 8192 data: 10 12
103962024-02-20T22:45:50.717Z07/30 Read block 96 len 8192
103972024-02-20T22:45:50.717Z08/30 Read block 148 len 40960
103982024-02-20T22:45:50.717Z09/30 Read block 483 len 28672
103992024-02-20T22:45:50.717Z10/30 Read block 64 len 40960
104002024-02-20T22:45:50.717Z11/30 Read block 457 len 20480
104012024-02-20T22:45:50.717Z12/30 Write block 543 len 20480 data: 8 7 8 8 9
104022024-02-20T22:45:50.717Z13/30 Read block 423 len 8192
104032024-02-20T22:45:50.717Z14/30 Write block 10 len 16384 data: 8 7 4 5
104042024-02-20T22:45:50.717Z15/30 Write block 275 len 12288 data: 4 3 4
104052024-02-20T22:45:50.717Z16/30 Read block 252 len 28672
104062024-02-20T22:45:50.721Z17/30 Read block 485 len 36864
104072024-02-20T22:45:50.721Z18/30 Read block 262 len 40960
104082024-02-20T22:45:50.721Z19/30 Write block 117 len 4096 data: 10
104092024-02-20T22:45:50.721Z20/30 Write block 218 len 4096 data: 3
104102024-02-20T22:45:50.722Z21/30 Write block 315 len 4096 data: 11
104112024-02-20T22:45:50.722Z22/30 Read block 72 len 24576
104122024-02-20T22:45:50.722Z23/30 Read block 197 len 4096
104132024-02-20T22:45:50.722Z24/30 Read block 469 len 40960
104142024-02-20T22:45:50.726Z25/30 Write block 76 len 20480 data: 4 5 5 5 5
104152024-02-20T22:45:50.726Z26/30 Read block 581 len 24576
104162024-02-20T22:45:50.726Z27/30 Read block 457 len 28672
104172024-02-20T22:45:50.726Z28/30 Read block 569 len 28672
104182024-02-20T22:45:50.726Z29/30 Read block 247 len 4096
104192024-02-20T22:45:50.731Z30/30 Write block 119 len 24576 data: 9 10 9 9 6 6
104202024-02-20T22:45:50.731Z----------------------------------------------------------------
104212024-02-20T22:45:50.731Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:26
104222024-02-20T22:45:50.732ZStates: Active Active Active
104232024-02-20T22:45:50.732ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104242024-02-20T22:45:50.732Z 5 Acked 1004 Write 5 Done Done Done false
104252024-02-20T22:45:50.732Z 6 Acked 1005 Write 2 Done Done Done false
104262024-02-20T22:45:50.732Z 7 Acked 1006 Read 2 Done Done Done false
104272024-02-20T22:45:50.732Z 8 Acked 1007 Read 10 Done Done Done false
104282024-02-20T22:45:50.732Z 9 Acked 1008 Read 7 Sent Done Done false
104292024-02-20T22:45:50.732Z 10 Acked 1009 Read 10 Sent Done Done false
104302024-02-20T22:45:50.732Z 11 Acked 1010 Read 5 Sent Done Done false
104312024-02-20T22:45:50.732Z 12 Acked 1011 Write 5 Sent Done Done false
104322024-02-20T22:45:50.732Z 13 Acked 1012 Read 2 Sent Done Done false
104332024-02-20T22:45:50.732Z 14 Acked 1013 Write 4 Sent Done Done false
104342024-02-20T22:45:50.732Z 15 Acked 1014 Write 3 Sent Done Done false
104352024-02-20T22:45:50.732Z 16 Acked 1015 Read 7 Sent Done Done false
104362024-02-20T22:45:50.732Z 17 Acked 1016 Read 9 Sent Done Done false
104372024-02-20T22:45:50.732Z 18 Acked 1017 Read 10 Sent Done Done false
104382024-02-20T22:45:50.732Z 19 Acked 1018 Write 1 Sent Done Done false
104392024-02-20T22:45:50.732Z 20 Acked 1019 Write 1 Sent Done Done false
104402024-02-20T22:45:50.732Z 21 Acked 1020 Write 1 Sent Done Done false
104412024-02-20T22:45:50.732Z 22 Acked 1021 Read 6 Sent Done Done false
104422024-02-20T22:45:50.732Z 23 Acked 1022 Read 1 Sent Done Done false
104432024-02-20T22:45:50.732Z 24 Acked 1023 Read 10 Sent Done Done false
104442024-02-20T22:45:50.732Z 25 Acked 1024 Write 5 Sent Done Done false
104452024-02-20T22:45:50.732Z 26 Acked 1025 Read 6 Sent Done Done false
104462024-02-20T22:45:50.732Z 27 Acked 1026 Read 7 Sent Done Done false
104472024-02-20T22:45:50.733Z 28 Acked 1027 Read 7 Sent Done Done false
104482024-02-20T22:45:50.733Z 29 Acked 1028 Read 1 Sent Done Done false
104492024-02-20T22:45:50.733Z 30 Acked 1029 Write 6 Sent Sent Sent false
104502024-02-20T22:45:50.733Z STATES DS:0 DS:1 DS:2 TOTAL
104512024-02-20T22:45:50.733Z New 0 0 0 0
104522024-02-20T22:45:50.733Z Sent 22 1 1 24
104532024-02-20T22:45:50.733Z Done 4 25 25 54
104542024-02-20T22:45:50.733Z Skipped 0 0 0 0
104552024-02-20T22:45:50.733Z Error 0 0 0 0
104562024-02-20T22:45:50.733ZLast Flush: 1003 1003 1003
104572024-02-20T22:45:50.733ZDownstairs last five completed: 1003 1002 1001 1000
104582024-02-20T22:45:50.733ZUpstairs last five completed: 30 29 28 27 26
104592024-02-20T22:45:50.733Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:50.250371815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1522}
104602024-02-20T22:45:50.733ZWrote out file "/tmp/test_repair_verify.data"
104612024-02-20T22:45:50.738Z
104622024-02-20T22:45:50.738ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104632024-02-20T22:45:50.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104642024-02-20T22:45:50.753Z 0 000-019 96 96 96 169 169 169 F T T <---
104652024-02-20T22:45:50.753Z 1 020-039 91 91 91 159 159 159 F F F
104662024-02-20T22:45:50.753Z 2 040-059 88 88 88 152 152 152 F F F
104672024-02-20T22:45:50.753Z 3 060-079 93 93 93 164 164 164 F T T <---
104682024-02-20T22:45:50.753Z 4 080-099 93 93 93 164 164 164 F T T <---
104692024-02-20T22:45:50.753Z 5 100-119 96 96 96 169 169 169 F T T <---
104702024-02-20T22:45:50.754Z 6 120-139 94 94 94 167 167 167 F T T <---
104712024-02-20T22:45:50.754Z 7 140-159 95 95 95 168 168 168 F F F
104722024-02-20T22:45:50.754Z 8 160-179 94 94 94 167 167 167 F F F
104732024-02-20T22:45:50.754Z 9 180-199 94 94 94 167 167 167 F F F
104742024-02-20T22:45:50.754Z 10 200-219 92 92 92 161 161 161 F T T <---
104752024-02-20T22:45:50.758Z 11 220-239 94 94 94 167 167 167 F F F
104762024-02-20T22:45:50.758Z 12 240-259 95 95 95 168 168 168 F F F
104772024-02-20T22:45:50.758Z 13 260-279 90 90 90 155 155 155 F T T <---
104782024-02-20T22:45:50.758Z 14 280-299 92 92 92 163 163 163 F F F
104792024-02-20T22:45:50.758Z 15 300-319 95 95 95 168 168 168 F T T <---
104802024-02-20T22:45:50.758Z 16 320-339 92 92 92 162 162 162 T T T
104812024-02-20T22:45:50.758Z 17 340-359 91 91 91 159 159 159 F F F
104822024-02-20T22:45:50.758Z 18 360-379 94 94 94 166 166 166 F F F
104832024-02-20T22:45:50.758Z 19 380-399 92 92 92 162 162 162 F F F
104842024-02-20T22:45:50.759Z 20 400-419 92 92 92 163 163 163 T T T
104852024-02-20T22:45:50.759Z 21 420-439 94 94 94 167 167 167 F F F
104862024-02-20T22:45:50.759Z 22 440-459 94 94 94 167 167 167 F F F
104872024-02-20T22:45:50.759Z 23 460-479 94 94 94 167 167 167 F F F
104882024-02-20T22:45:50.759Z 24 480-499 94 94 94 167 167 167 F F F
104892024-02-20T22:45:50.759Z 25 500-519 92 92 92 162 162 162 F F F
104902024-02-20T22:45:50.759Z 26 520-539 94 94 94 167 167 167 F F F
104912024-02-20T22:45:50.759Z 27 540-559 94 94 94 167 167 167 F T T <---
104922024-02-20T22:45:50.759Z 28 560-579 93 93 93 164 164 164 F F F
104932024-02-20T22:45:50.759Z 29 580-599 88 88 88 151 151 151 F F F
104942024-02-20T22:45:50.759ZMax gen: 96, Max flush: 169
104952024-02-20T22:45:50.759ZError: Difference in extent metadata found!
104962024-02-20T22:45:50.759ZA Difference in extent metadata is expected here
104972024-02-20T22:45:50.759ZOn loop 48
104982024-02-20T22:45:50.759Z
104992024-02-20T22:45:50.763ZVerifying data now
105002024-02-20T22:45:55.326ZNo change after verify
105012024-02-20T22:45:55.326ZLoop: 48 Downstairs dump after verify (and repair):
105022024-02-20T22:45:55.344ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105032024-02-20T22:45:55.344Z 0 000-019 97 97 97 170 170 170 F F F
105042024-02-20T22:45:55.344Z 1 020-039 91 91 91 159 159 159 F F F
105052024-02-20T22:45:55.344Z 2 040-059 88 88 88 152 152 152 F F F
105062024-02-20T22:45:55.344Z 3 060-079 97 97 97 170 170 170 F F F
105072024-02-20T22:45:55.344Z 4 080-099 97 97 97 170 170 170 F F F
105082024-02-20T22:45:55.344Z 5 100-119 97 97 97 170 170 170 F F F
105092024-02-20T22:45:55.344Z 6 120-139 97 97 97 170 170 170 F F F
105102024-02-20T22:45:55.344Z 7 140-159 95 95 95 168 168 168 F F F
105112024-02-20T22:45:55.353Z 8 160-179 94 94 94 167 167 167 F F F
105122024-02-20T22:45:55.353Z 9 180-199 94 94 94 167 167 167 F F F
105132024-02-20T22:45:55.353Z 10 200-219 97 97 97 170 170 170 F F F
105142024-02-20T22:45:55.353Z 11 220-239 94 94 94 167 167 167 F F F
105152024-02-20T22:45:55.353Z 12 240-259 95 95 95 168 168 168 F F F
105162024-02-20T22:45:55.353Z 13 260-279 97 97 97 170 170 170 F F F
105172024-02-20T22:45:55.353Z 14 280-299 92 92 92 163 163 163 F F F
105182024-02-20T22:45:55.353Z 15 300-319 97 97 97 170 170 170 F F F
105192024-02-20T22:45:55.353Z 16 320-339 97 97 97 170 170 170 F F F
105202024-02-20T22:45:55.353Z 17 340-359 91 91 91 159 159 159 F F F
105212024-02-20T22:45:55.353Z 18 360-379 94 94 94 166 166 166 F F F
105222024-02-20T22:45:55.353Z 19 380-399 92 92 92 162 162 162 F F F
105232024-02-20T22:45:55.353Z 20 400-419 97 97 97 170 170 170 F F F
105242024-02-20T22:45:55.353Z 21 420-439 94 94 94 167 167 167 F F F
105252024-02-20T22:45:55.354Z 22 440-459 94 94 94 167 167 167 F F F
105262024-02-20T22:45:55.354Z 23 460-479 94 94 94 167 167 167 F F F
105272024-02-20T22:45:55.354Z 24 480-499 94 94 94 167 167 167 F F F
105282024-02-20T22:45:55.354Z 25 500-519 92 92 92 162 162 162 F F F
105292024-02-20T22:45:55.354Z 26 520-539 94 94 94 167 167 167 F F F
105302024-02-20T22:45:55.354Z 27 540-559 97 97 97 170 170 170 F F F
105312024-02-20T22:45:55.354Z 28 560-579 93 93 93 164 164 164 F F F
105322024-02-20T22:45:55.354Z 29 580-599 88 88 88 151 151 151 F F F
105332024-02-20T22:45:55.354ZMax gen: 97, Max flush: 170
105342024-02-20T22:45:55.354Z
105352024-02-20T22:45:55.354ZBegin loop 49
105362024-02-20T22:45:55.354ZDownstairs to restart: 0
105372024-02-20T22:45:55.361Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.883819286Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530}
105382024-02-20T22:45:55.361Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884114936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530}
105392024-02-20T22:45:55.368Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884153974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530}
105402024-02-20T22:45:55.368Z{"msg":"Crucible 1d1ee0e2-c521-4372-9114-69d4f65cf4c2 has session id: a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884189353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105412024-02-20T22:45:55.368Z{"msg":"Upstairs opts: Upstairs UUID: 1d1ee0e2-c521-4372-9114-69d4f65cf4c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884235042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530Crucible runtime is spawned
105422024-02-20T22:45:55.368Z,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105432024-02-20T22:45:55.368Z{"msg":"Crucible stats registered with UUID: 1d1ee0e2-c521-4372-9114-69d4f65cf4c2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.8842682Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105442024-02-20T22:45:55.368Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884324098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530}
105452024-02-20T22:45:55.368Z{"msg":"1d1ee0e2-c521-4372-9114-69d4f65cf4c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884469183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105462024-02-20T22:45:55.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884500982Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105472024-02-20T22:45:55.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884536051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105482024-02-20T22:45:55.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.88456032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105492024-02-20T22:45:55.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884594449Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105502024-02-20T22:45:55.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884630477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105512024-02-20T22:45:55.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884646987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105522024-02-20T22:45:55.368Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884679235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105532024-02-20T22:45:55.368Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884701065Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105542024-02-20T22:45:55.368Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884733834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105552024-02-20T22:45:55.368Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884895658Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105562024-02-20T22:45:55.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884944386Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105572024-02-20T22:45:55.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.884974915Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105582024-02-20T22:45:55.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.885697769Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105592024-02-20T22:45:55.369Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.885735598Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105602024-02-20T22:45:55.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.885751487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105612024-02-20T22:45:55.369Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.885791666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105622024-02-20T22:45:55.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.885816055Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105632024-02-20T22:45:55.369Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.885844094Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105642024-02-20T22:45:55.369Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886325717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105652024-02-20T22:45:55.369Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886368525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105662024-02-20T22:45:55.369Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886400194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105672024-02-20T22:45:55.369Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886893756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105682024-02-20T22:45:55.369Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886930835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105692024-02-20T22:45:55.369Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886948314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105702024-02-20T22:45:55.369Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886965854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105712024-02-20T22:45:55.369Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.886990093Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105722024-02-20T22:45:55.369Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887013122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105732024-02-20T22:45:55.369Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887045211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105742024-02-20T22:45:55.369Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.88707674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105752024-02-20T22:45:55.369Z{"msg":"[0]R flush_numbers[0..12]: [170, 159, 152, 170, 170, 170, 170, 168, 167, 167, 170, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887110728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105762024-02-20T22:45:55.369Z{"msg":"[0]R generation[0..12]: [97, 91, 88, 97, 97, 97, 97, 95, 94, 94, 97, 94]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
105772024-02-20T22:45:55.369Z,"time":"2024-02-20T22:45:54.887146017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105782024-02-20T22:45:55.369Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887183066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105792024-02-20T22:45:55.369Z{"msg":"[1]R flush_numbers[0..12]: [170, 159, 152, 170, 170, 170, 170, 168, 167, 167, 170, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887201245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105802024-02-20T22:45:55.370Z{"msg":"[1]R generation[0..12]: [97, 91, 88, 97, 97, 97, 97, 95, 94, 94, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887215675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105812024-02-20T22:45:55.370Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887238544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105822024-02-20T22:45:55.370Z{"msg":"[2]R flush_numbers[0..12]: [170, 159, 152, 170, 170, 170, 170, 168, 167, 167, 170, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887277193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105832024-02-20T22:45:55.370Z{"msg":"[2]R generation[0..12]: [97, 91, 88, 97, 97, 97, 97, 95, 94, 94, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887300862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105842024-02-20T22:45:55.370Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887315641Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105852024-02-20T22:45:55.370Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.88734252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105862024-02-20T22:45:55.370Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887368099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105872024-02-20T22:45:55.370Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887402248Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105882024-02-20T22:45:55.370Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887418368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105892024-02-20T22:45:55.370Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887441997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105902024-02-20T22:45:55.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887464316Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105912024-02-20T22:45:55.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887501314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105922024-02-20T22:45:55.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887521044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105932024-02-20T22:45:55.370Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887542523Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105942024-02-20T22:45:55.370Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887563552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105952024-02-20T22:45:55.370Z{"msg":"1d1ee0e2-c521-4372-9114-69d4f65cf4c2 is now active with session: a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.887600441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105962024-02-20T22:45:55.370Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.88761707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530,"session_id":"a8c2ac8f-ed76-4ba5-a6a9-5f1fddd8adba"}
105972024-02-20T22:45:55.370Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:54.88763821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530}
105982024-02-20T22:45:55.370ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105992024-02-20T22:45:55.374ZLoading write count information from file "/tmp/test_repair_verify.data"
106002024-02-20T22:45:55.374ZRun Repair workload
106012024-02-20T22:45:55.374Z01/30 Write block 324 len 24576 data: 4 7 8 8 8 8
106022024-02-20T22:45:55.374Z02/30 Read block 296 len 20480
106032024-02-20T22:45:55.374Z03/30 Write block 13 len 32768 data: 6 4 5 8 8 7 7 7
106042024-02-20T22:45:55.374Z04/30 Flush
106052024-02-20T22:45:55.374Z05/30 Flush
106062024-02-20T22:45:55.375Z06/30 Write block 558 len 4096 data: 5
106072024-02-20T22:45:55.375Z07/30 Write block 456 len 32768 data: 14 15 16 13 11 12 13 13
106082024-02-20T22:45:55.375Z08/30 Flush
106092024-02-20T22:45:55.378Z09/30 Write block 123 len 32768 data: 7 7 5 4 8 8 8 8
106102024-02-20T22:45:55.378Z10/30 Write block 577 len 32768 data: 9 10 12 13 11 11 10 8
106112024-02-20T22:45:55.378Z11/30 Write block 317 len 40960 data: 12 9 10 12 10 12 8 5 8 9
106122024-02-20T22:45:55.382Z12/30 Read block 518 len 36864
106132024-02-20T22:45:55.382Z13/30 Write block 250 len 8192 data: 5 4
106142024-02-20T22:45:55.382Z14/30 Read block 386 len 28672
106152024-02-20T22:45:55.382Z15/30 Read block 454 len 36864
106162024-02-20T22:45:55.386Z16/30 Read block 544 len 8192
106172024-02-20T22:45:55.386Z17/30 Write block 224 len 16384 data: 5 4 5 6
106182024-02-20T22:45:55.386Z18/30 Write block 571 len 20480 data: 3 3 5 5 6
106192024-02-20T22:45:55.386Z19/30 Read block 519 len 4096
106202024-02-20T22:45:55.386Z20/30 Read block 574 len 4096
106212024-02-20T22:45:55.390Z21/30 Write block 70 len 36864 data: 10 10 9 7 8 5 5 6 6
106222024-02-20T22:45:55.390Z22/30 Read block 487 len 16384
106232024-02-20T22:45:55.390Z23/30 Read block 496 len 32768
106242024-02-20T22:45:55.390Z24/30 Write block 70 len 24576 data: 11 11 10 8 9 6
106252024-02-20T22:45:55.394Z25/30 Write block 16 len 20480 data: 9 9 8 8 8
106262024-02-20T22:45:55.394Z26/30 Read block 588 len 32768
106272024-02-20T22:45:55.394Z27/30 Write block 67 len 20480 data: 8 9 10 12 12
106282024-02-20T22:45:55.394Z28/30 Write block 482 len 36864 data: 13 13 11 9 7 6 7 5 5
106292024-02-20T22:45:55.394Z29/30 Read block 576 len 12288
106302024-02-20T22:45:55.399Z30/30 Write block 558 len 36864 data: 6 4 4 7 5 6 8 7 8
106312024-02-20T22:45:55.399Z----------------------------------------------------------------
106322024-02-20T22:45:55.399Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:22
106332024-02-20T22:45:55.399ZStates: Active Active Active
106342024-02-20T22:45:55.399ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106352024-02-20T22:45:55.399Z 9 Acked 1008 Write 8 Done Done Done false
106362024-02-20T22:45:55.399Z 10 Acked 1009 Write 8 Sent Done Done false
106372024-02-20T22:45:55.399Z 11 Acked 1010 Write 10 Sent Done Done false
106382024-02-20T22:45:55.399Z 12 Acked 1011 Read 9 Sent Done Done false
106392024-02-20T22:45:55.399Z 13 Acked 1012 Write 2 Sent Done Done false
106402024-02-20T22:45:55.399Z 14 Acked 1013 Read 7 Sent Done Done false
106412024-02-20T22:45:55.399Z 15 Acked 1014 Read 9 Sent Done Done false
106422024-02-20T22:45:55.399Z 16 Acked 1015 Read 2 Sent Done Done false
106432024-02-20T22:45:55.399Z 17 Acked 1016 Write 4 Sent Done Done false
106442024-02-20T22:45:55.399Z 18 Acked 1017 Write 5 Sent Done Done false
106452024-02-20T22:45:55.400Z 19 Acked 1018 Read 1 Sent Done Done false
106462024-02-20T22:45:55.400Z 20 Acked 1019 Read 1 Sent Done Done false
106472024-02-20T22:45:55.400Z 21 Acked 1020 Write 9 Sent Done Done false
106482024-02-20T22:45:55.400Z 22 Acked 1021 Read 4 Sent Done Done false
106492024-02-20T22:45:55.400Z 23 Acked 1022 Read 8 Sent Done Done false
106502024-02-20T22:45:55.400Z 24 Acked 1023 Write 6 Sent Done Done false
106512024-02-20T22:45:55.400Z 25 Acked 1024 Write 5 Sent Done Done false
106522024-02-20T22:45:55.400Z 26 Acked 1025 Read 8 Sent Done Done false
106532024-02-20T22:45:55.400Z 27 Acked 1026 Write 5 Sent Done Done false
106542024-02-20T22:45:55.400Z 28 Acked 1027 Write 9 Sent Done Done false
106552024-02-20T22:45:55.400Z 29 Acked 1028 Read 3 Sent Done Done false
106562024-02-20T22:45:55.400Z 30 Acked 1029 Write 9 Sent Sent Sent false
106572024-02-20T22:45:55.400Z STATES DS:0 DS:1 DS:2 TOTAL
106582024-02-20T22:45:55.400Z New 0 0 0 0
106592024-02-20T22:45:55.400Z Sent 21 1 1 23
106602024-02-20T22:45:55.400Z Done 1 21 21 43
106612024-02-20T22:45:55.400Z Skipped 0 0 0 0
106622024-02-20T22:45:55.400Z Error 0 0 0 0
106632024-02-20T22:45:55.400ZLast Flush: 1007 1007 1007
106642024-02-20T22:45:55.400ZDownstairs last five completed: 1007 1006 1005 1004 1003
106652024-02-20T22:45:55.400ZUpstairs last five completed: 30 29 28 27 26
106662024-02-20T22:45:55.400Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:54.919368864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1530}
106672024-02-20T22:45:55.400ZWrote out file "/tmp/test_repair_verify.data"
106682024-02-20T22:45:55.405Z
106692024-02-20T22:45:55.408ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106702024-02-20T22:45:55.832ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106712024-02-20T22:45:55.832Z 0 000-019 98 98 98 171 171 171 F T T <---
106722024-02-20T22:45:55.832Z 1 020-039 98 98 98 171 171 171 F T T <---
106732024-02-20T22:45:55.832Z 2 040-059 88 88 88 152 152 152 F F F
106742024-02-20T22:45:55.832Z 3 060-079 97 97 97 170 170 170 F T T <---
106752024-02-20T22:45:55.832Z 4 080-099 97 97 97 170 170 170 F F F
106762024-02-20T22:45:55.832Z 5 100-119 97 97 97 170 170 170 F F F
106772024-02-20T22:45:55.832Z 6 120-139 97 97 97 170 170 170 T T T
106782024-02-20T22:45:55.837Z 7 140-159 95 95 95 168 168 168 F F F
106792024-02-20T22:45:55.837Z 8 160-179 94 94 94 167 167 167 F F F
106802024-02-20T22:45:55.837Z 9 180-199 94 94 94 167 167 167 F F F
106812024-02-20T22:45:55.837Z 10 200-219 97 97 97 170 170 170 F F F
106822024-02-20T22:45:55.837Z 11 220-239 94 94 94 167 167 167 F T T <---
106832024-02-20T22:45:55.838Z 12 240-259 95 95 95 168 168 168 F T T <---
106842024-02-20T22:45:55.838Z 13 260-279 97 97 97 170 170 170 F F F
106852024-02-20T22:45:55.838Z 14 280-299 92 92 92 163 163 163 F F F
106862024-02-20T22:45:55.838Z 15 300-319 97 97 97 170 170 170 F T T <---
106872024-02-20T22:45:55.838Z 16 320-339 98 98 98 171 171 171 F T T <---
106882024-02-20T22:45:55.838Z 17 340-359 91 91 91 159 159 159 F F F
106892024-02-20T22:45:55.838Z 18 360-379 94 94 94 166 166 166 F F F
106902024-02-20T22:45:55.838Z 19 380-399 92 92 92 162 162 162 F F F
106912024-02-20T22:45:55.838Z 20 400-419 97 97 97 170 170 170 F F F
106922024-02-20T22:45:55.838Z 21 420-439 94 94 94 167 167 167 F F F
106932024-02-20T22:45:55.838Z 22 440-459 98 98 98 173 173 173 F F F
106942024-02-20T22:45:55.838Z 23 460-479 98 98 98 173 173 173 F F F
106952024-02-20T22:45:55.838Z 24 480-499 94 94 94 167 167 167 F T T <---
106962024-02-20T22:45:55.838Z 25 500-519 92 92 92 162 162 162 F F F
106972024-02-20T22:45:55.838Z 26 520-539 94 94 94 167 167 167 F F F
106982024-02-20T22:45:55.838Z 27 540-559 98 98 98 173 173 173 F T T <---
106992024-02-20T22:45:55.838Z 28 560-579 93 93 93 164 164 164 F T T <---
107002024-02-20T22:45:55.838Z 29 580-599 88 88 88 151 151 151 F T T <---
107012024-02-20T22:45:55.838ZMax gen: 98, Max flush: 173
107022024-02-20T22:45:55.838ZError: Difference in extent metadata found!
107032024-02-20T22:45:55.838ZA Difference in extent metadata is expected here
107042024-02-20T22:45:55.838ZOn loop 49
107052024-02-20T22:45:55.838Z
107062024-02-20T22:45:55.838ZVerifying data now
107072024-02-20T22:46:00.077ZNo change after verify
107082024-02-20T22:46:00.077ZLoop: 49 Downstairs dump after verify (and repair):
107092024-02-20T22:46:00.101ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107102024-02-20T22:46:00.101Z 0 000-019 99 99 99 174 174 174 F F F
107112024-02-20T22:46:00.101Z 1 020-039 99 99 99 174 174 174 F F F
107122024-02-20T22:46:00.101Z 2 040-059 88 88 88 152 152 152 F F F
107132024-02-20T22:46:00.101Z 3 060-079 99 99 99 174 174 174 F F F
107142024-02-20T22:46:00.101Z 4 080-099 97 97 97 170 170 170 F F F
107152024-02-20T22:46:00.101Z 5 100-119 97 97 97 170 170 170 F F F
107162024-02-20T22:46:00.101Z 6 120-139 99 99 99 174 174 174 F F F
107172024-02-20T22:46:00.101Z 7 140-159 95 95 95 168 168 168 F F F
107182024-02-20T22:46:00.101Z 8 160-179 94 94 94 167 167 167 F F F
107192024-02-20T22:46:00.101Z 9 180-199 94 94 94 167 167 167 F F F
107202024-02-20T22:46:00.101Z 10 200-219 97 97 97 170 170 170 F F F
107212024-02-20T22:46:00.101Z 11 220-239 99 99 99 174 174 174 F F F
107222024-02-20T22:46:00.101Z 12 240-259 99 99 99 174 174 174 F F F
107232024-02-20T22:46:00.101Z 13 260-279 97 97 97 170 170 170 F F F
107242024-02-20T22:46:00.101Z 14 280-299 92 92 92 163 163 163 F F F
107252024-02-20T22:46:00.101Z 15 300-319 99 99 99 174 174 174 F F F
107262024-02-20T22:46:00.101Z 16 320-339 99 99 99 174 174 174 F F F
107272024-02-20T22:46:00.101Z 17 340-359 91 91 91 159 159 159 F F F
107282024-02-20T22:46:00.101Z 18 360-379 94 94 94 166 166 166 F F F
107292024-02-20T22:46:00.102Z 19 380-399 92 92 92 162 162 162 F F F
107302024-02-20T22:46:00.102Z 20 400-419 97 97 97 170 170 170 F F F
107312024-02-20T22:46:00.102Z 21 420-439 94 94 94 167 167 167 F F F
107322024-02-20T22:46:00.102Z 22 440-459 98 98 98 173 173 173 F F F
107332024-02-20T22:46:00.102Z 23 460-479 98 98 98 173 173 173 F F F
107342024-02-20T22:46:00.102Z 24 480-499 99 99 99 174 174 174 F F F
107352024-02-20T22:46:00.102Z 25 500-519 92 92 92 162 162 162 F F F
107362024-02-20T22:46:00.102Z 26 520-539 94 94 94 167 167 167 F F F
107372024-02-20T22:46:00.102Z 27 540-559 99 99 99 174 174 174 F F F
107382024-02-20T22:46:00.102Z 28 560-579 99 99 99 174 174 174 F F F
107392024-02-20T22:46:00.102Z 29 580-599 99 99 99 174 174 174 F F F
107402024-02-20T22:46:00.102ZMax gen: 99, Max flush: 174
107412024-02-20T22:46:00.102Z
107422024-02-20T22:46:00.102ZBegin loop 50
107432024-02-20T22:46:00.102ZDownstairs to restart: 0
107442024-02-20T22:46:00.102Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.590597234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538}
107452024-02-20T22:46:00.102Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.590885384Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538}
107462024-02-20T22:46:00.102ZCrucible runtime is spawned
107472024-02-20T22:46:00.102Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.590931632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538}
107482024-02-20T22:46:00.102Z{"msg":"Crucible 5fcfe3ac-a488-4b1d-8fbe-caa6cf4d80d2 has session id: bc248ab4-5b56-4409-b975-91dd4a7b7b59","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59097545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107492024-02-20T22:46:00.102Z{"msg":"Upstairs opts: Upstairs UUID: 5fcfe3ac-a488-4b1d-8fbe-caa6cf4d80d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591028858Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107502024-02-20T22:46:00.102Z{"msg":"Crucible stats registered with UUID: 5fcfe3ac-a488-4b1d-8fbe-caa6cf4d80d2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591083417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107512024-02-20T22:46:00.102Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591114585Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538}
107522024-02-20T22:46:00.102Z{"msg":"5fcfe3ac-a488-4b1d-8fbe-caa6cf4d80d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591140124Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107532024-02-20T22:46:00.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591183193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107542024-02-20T22:46:00.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591233631Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107552024-02-20T22:46:00.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59125854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107562024-02-20T22:46:00.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591288529Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107572024-02-20T22:46:00.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591307009Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107582024-02-20T22:46:00.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591331317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107592024-02-20T22:46:00.103Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591385666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107602024-02-20T22:46:00.103Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591436594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107612024-02-20T22:46:00.103Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591464903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107622024-02-20T22:46:00.103Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591493382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107632024-02-20T22:46:00.103Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.5915322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107642024-02-20T22:46:00.103Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.591567889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107652024-02-20T22:46:00.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592182917Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107662024-02-20T22:46:00.103Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592220506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107672024-02-20T22:46:00.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592237105Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107682024-02-20T22:46:00.103Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592251555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107692024-02-20T22:46:00.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592264914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107702024-02-20T22:46:00.103Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592296413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107712024-02-20T22:46:00.103Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592816324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107722024-02-20T22:46:00.103Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59294131Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107732024-02-20T22:46:00.103Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.592970679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107742024-02-20T22:46:00.103Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593378554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107752024-02-20T22:46:00.104Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593443492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107762024-02-20T22:46:00.104Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59348763Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107772024-02-20T22:46:00.104Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59350944Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107782024-02-20T22:46:00.104Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593540338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107792024-02-20T22:46:00.104Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593563198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107802024-02-20T22:46:00.104Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593591767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107812024-02-20T22:46:00.104Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593612846Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107822024-02-20T22:46:00.104Z{"msg":"[0]R flush_numbers[0..12]: [174, 174, 152, 174, 170, 170, 174, 168, 167, 167, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593642525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107832024-02-20T22:46:00.104Z{"msg":"[0]R generation[0..12]: [99, 99, 88, 99, 97, 97, 99, 95, 94, 94, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593673504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
107842024-02-20T22:46:00.104Zbc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107852024-02-20T22:46:00.104Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593713772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107862024-02-20T22:46:00.104Z{"msg":"[1]R flush_numbers[0..12]: [174, 174, 152, 174, 170, 170, 174, 168, 167, 167, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593748241Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107872024-02-20T22:46:00.104Z{"msg":"[1]R generation[0..12]: [99, 99, 88, 99, 97, 97, 99, 95, 94, 94, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59378178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107882024-02-20T22:46:00.104Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593816049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107892024-02-20T22:46:00.104Z{"msg":"[2]R flush_numbers[0..12]: [174, 174, 152, 174, 170, 170, 174, 168, 167, 167, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593848878Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107902024-02-20T22:46:00.104Z{"msg":"[2]R generation[0..12]: [99, 99, 88, 99, 97, 97, 99, 95, 94, 94, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593881666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107912024-02-20T22:46:00.104Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593914845Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107922024-02-20T22:46:00.104Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593946784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107932024-02-20T22:46:00.104Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593978513Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107942024-02-20T22:46:00.104Z{"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.593996012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107952024-02-20T22:46:00.104Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594026681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107962024-02-20T22:46:00.104Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59404478Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107972024-02-20T22:46:00.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.59407331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107982024-02-20T22:46:00.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594089159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
107992024-02-20T22:46:00.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594119588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
108002024-02-20T22:46:00.104Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594137867Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108012024-02-20T22:46:00.105Zsession_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
108022024-02-20T22:46:00.105Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594168436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
108032024-02-20T22:46:00.105Z{"msg":"5fcfe3ac-a488-4b1d-8fbe-caa6cf4d80d2 is now active with session: bc248ab4-5b56-4409-b975-91dd4a7b7b59","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594187755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
108042024-02-20T22:46:00.105Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594216444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538,"session_id":"bc248ab4-5b56-4409-b975-91dd4a7b7b59"}
108052024-02-20T22:46:00.105Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:45:59.594231374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538}
108062024-02-20T22:46:00.105ZLoading write count information from file "/tmp/test_repair_verify.data"
108072024-02-20T22:46:00.105ZRun Repair workload
108082024-02-20T22:46:00.105Z01/30 Write block 556 len 8192 data: 6 5
108092024-02-20T22:46:00.136Z02/30 Write block 553 len 28672 data: 5 5 6 7 6 7 5
108102024-02-20T22:46:00.136Z03/30 Read block 559 len 12288
108112024-02-20T22:46:00.136Z04/30 Write block 122 len 32768 data: 10 8 8 6 5 9 9 9
108122024-02-20T22:46:00.136Z05/30 Read block 306 len 40960
108132024-02-20T22:46:00.136Z06/30 Write block 59 len 4096 data: 7
108142024-02-20T22:46:00.136Z07/30 Read block 69 len 16384
108152024-02-20T22:46:00.136Z08/30 Write block 563 len 12288 data: 7 9 8
108162024-02-20T22:46:00.136Z09/30 Flush
108172024-02-20T22:46:00.136Z10/30 Write block 104 len 4096 data: 4
108182024-02-20T22:46:00.136Z11/30 Write block 16 len 32768 data: 10 10 9 9 9 6 7 7
108192024-02-20T22:46:00.136Z12/30 Read block 409 len 36864
108202024-02-20T22:46:00.136Z13/30 Read block 532 len 4096
108212024-02-20T22:46:00.137Z14/30 Write block 337 len 20480 data: 10 10 11 10 11
108222024-02-20T22:46:00.137Z15/30 Write block 130 len 16384 data: 9 8 8 9
108232024-02-20T22:46:00.137Z16/30 Write block 514 len 28672 data: 9 9 7 8 6 5 7
108242024-02-20T22:46:00.137Z17/30 Read block 387 len 4096
108252024-02-20T22:46:00.137Z18/30 Read block 346 len 28672
108262024-02-20T22:46:00.137Z19/30 Write block 252 len 8192 data: 4 7
108272024-02-20T22:46:00.137Z20/30 Read block 488 len 16384
108282024-02-20T22:46:00.137Z21/30 Flush
108292024-02-20T22:46:00.137Z22/30 Write block 416 len 24576 data: 9 7 8 11 11 9
108302024-02-20T22:46:00.137Z23/30 Write block 591 len 36864 data: 12 11 10 11 11 10 9 7 3
108312024-02-20T22:46:00.137Z24/30 Read block 41 len 32768
108322024-02-20T22:46:00.137Z25/30 Write block 267 len 20480 data: 9 9 9 7 6
108332024-02-20T22:46:00.137Z26/30 Read block 385 len 4096
108342024-02-20T22:46:00.137Z27/30 Read block 57 len 16384
108352024-02-20T22:46:00.137Z28/30 Read block 83 len 24576
108362024-02-20T22:46:00.137Z29/30 Read block 330 len 32768
108372024-02-20T22:46:00.137Z30/30 Read block 498 len 8192
108382024-02-20T22:46:00.137Z----------------------------------------------------------------
108392024-02-20T22:46:00.137Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
108402024-02-20T22:46:00.137ZStates: Active Active Active
108412024-02-20T22:46:00.137ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108422024-02-20T22:46:00.137Z 1 Acked 1000 Write 2 Done Done Done false
108432024-02-20T22:46:00.137Z 2 Acked 1001 Write 7 Done Done Done false
108442024-02-20T22:46:00.138Z 3 Acked 1002 Read 3 Done Done Done false
108452024-02-20T22:46:00.138Z 4 Acked 1003 Write 8 Sent Done Done false
108462024-02-20T22:46:00.138Z 5 Acked 1004 Read 10 Sent Done Done false
108472024-02-20T22:46:00.138Z 6 Acked 1005 Write 1 Sent Done Done false
108482024-02-20T22:46:00.138Z 7 Acked 1006 Read 4 Sent Done Done false
108492024-02-20T22:46:00.138Z 8 Acked 1007 Write 3 Sent Done Done false
108502024-02-20T22:46:00.138Z 9 Acked 1008 Flush 0 Sent Done Done false
108512024-02-20T22:46:00.138Z 10 Acked 1009 Write 1 Sent Done Done false
108522024-02-20T22:46:00.138Z 11 Acked 1010 Write 8 Sent Done Done false
108532024-02-20T22:46:00.138Z 12 Acked 1011 Read 9 Sent Done Done false
108542024-02-20T22:46:00.138Z 13 Acked 1012 Read 1 Sent Done Done false
108552024-02-20T22:46:00.138Z 14 Acked 1013 Write 5 Sent Done Done false
108562024-02-20T22:46:00.138Z 15 Acked 1014 Write 4 Sent Done Done false
108572024-02-20T22:46:00.138Z 16 Acked 1015 Write 7 Sent Done Done false
108582024-02-20T22:46:00.138Z 17 Acked 1016 Read 1 Sent Done Done false
108592024-02-20T22:46:00.138Z 18 Acked 1017 Read 7 Sent Done Done false
108602024-02-20T22:46:00.138Z 19 Acked 1018 Write 2 Sent Done Done false
108612024-02-20T22:46:00.138Z 20 Acked 1019 Read 4 Sent Done Done false
108622024-02-20T22:46:00.138Z 21 Acked 1020 Flush 0 Sent Done Done false
108632024-02-20T22:46:00.138Z 22 Acked 1021 Write 6 Sent Done Done false
108642024-02-20T22:46:00.138Z 23 Acked 1022 Write 9 Sent Done Done false
108652024-02-20T22:46:00.138Z 24 Acked 1023 Read 8 Sent Done Done false
108662024-02-20T22:46:00.139Z 25 Acked 1024 Write 5 Sent Done Done false
108672024-02-20T22:46:00.139Z 26 Acked 1025 Read 1 Sent Done Done false
108682024-02-20T22:46:00.139Z 27 Acked 1026 Read 4 Sent Done Done false
108692024-02-20T22:46:00.139Z 28 Acked 1027 Read 6 Sent Done Done false
108702024-02-20T22:46:00.139Z 29 Acked 1028 Read 8 Sent Done Done false
108712024-02-20T22:46:00.139Z 30 Acked 1029 Read 2 Sent Done Done false
108722024-02-20T22:46:00.139Z STATES DS:0 DS:1 DS:2 TOTAL
108732024-02-20T22:46:00.139Z New 0 0 0 0
108742024-02-20T22:46:00.139Z Sent 27 0 0 27
108752024-02-20T22:46:00.139Z Done 3 30 30 63
108762024-02-20T22:46:00.139Z Skipped 0 0 0 0
108772024-02-20T22:46:00.139Z Error 0 0 0 0
108782024-02-20T22:46:00.139ZLast Flush: 0 1020 1020
108792024-02-20T22:46:00.139ZDownstairs last five completed:
108802024-02-20T22:46:00.139ZUpstairs last five completed: 30 29 28 27 26
108812024-02-20T22:46:00.139Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:45:59.623653491Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1538}
108822024-02-20T22:46:00.139ZWrote out file "/tmp/test_repair_verify.data"
108832024-02-20T22:46:00.139Z
108842024-02-20T22:46:00.139ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108852024-02-20T22:46:00.139ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108862024-02-20T22:46:00.139Z 0 000-019 99 100 100 174 176 176 F F F <---
108872024-02-20T22:46:00.139Z 1 020-039 99 100 100 174 176 176 F F F <---
108882024-02-20T22:46:00.139Z 2 040-059 88 100 100 152 175 175 F F F <---
108892024-02-20T22:46:00.140Z 3 060-079 99 99 99 174 174 174 F F F
108902024-02-20T22:46:00.140Z 4 080-099 97 97 97 170 170 170 F F F
108912024-02-20T22:46:00.140Z 5 100-119 97 100 100 170 176 176 F F F <---
108922024-02-20T22:46:00.140Z 6 120-139 99 100 100 174 176 176 F F F <---
108932024-02-20T22:46:00.140Z 7 140-159 95 95 95 168 168 168 F F F
108942024-02-20T22:46:00.140Z 8 160-179 94 94 94 167 167 167 F F F
108952024-02-20T22:46:00.140Z 9 180-199 94 94 94 167 167 167 F F F
108962024-02-20T22:46:00.140Z 10 200-219 97 97 97 170 170 170 F F F
108972024-02-20T22:46:00.140Z 11 220-239 99 99 99 174 174 174 F F F
108982024-02-20T22:46:00.140Z 12 240-259 99 100 100 174 176 176 F F F <---
108992024-02-20T22:46:00.140Z 13 260-279 97 97 97 170 170 170 F T T <---
109002024-02-20T22:46:00.140Z 14 280-299 92 92 92 163 163 163 F F F
109012024-02-20T22:46:00.140Z 15 300-319 99 99 99 174 174 174 F F F
109022024-02-20T22:46:00.140Z 16 320-339 99 100 100 174 176 176 F F F <---
109032024-02-20T22:46:00.140Z 17 340-359 91 100 100 159 176 176 F F F <---
109042024-02-20T22:46:00.140Z 18 360-379 94 94 94 166 166 166 F F F
109052024-02-20T22:46:00.140Z 19 380-399 92 92 92 162 162 162 F F F
109062024-02-20T22:46:00.140Z 20 400-419 97 97 97 170 170 170 F T T <---
109072024-02-20T22:46:00.140Z 21 420-439 94 94 94 167 167 167 F T T <---
109082024-02-20T22:46:00.140Z 22 440-459 98 98 98 173 173 173 F F F
109092024-02-20T22:46:00.153Z 23 460-479 98 98 98 173 173 173 F F F
109102024-02-20T22:46:00.153Z 24 480-499 99 99 99 174 174 174 F F F
109112024-02-20T22:46:00.153Z 25 500-519 92 100 100 162 176 176 F F F <---
109122024-02-20T22:46:00.153Z 26 520-539 94 100 100 167 176 176 F F F <---
109132024-02-20T22:46:00.153Z 27 540-559 99 100 100 174 175 175 T F F <---
109142024-02-20T22:46:00.153Z 28 560-579 99 100 100 174 175 175 F F F <---
109152024-02-20T22:46:00.153Z 29 580-599 99 99 99 174 174 174 F T T <---
109162024-02-20T22:46:00.153ZMax gen: 100, Max flush: 176
109172024-02-20T22:46:00.153ZError: Difference in extent metadata found!
109182024-02-20T22:46:00.153ZA Difference in extent metadata is expected here
109192024-02-20T22:46:00.153ZOn loop 50
109202024-02-20T22:46:00.153Z
109212024-02-20T22:46:00.153ZVerifying data now
109222024-02-20T22:46:04.874ZNo change after verify
109232024-02-20T22:46:04.874ZLoop: 50 Downstairs dump after verify (and repair):
109242024-02-20T22:46:04.892ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109252024-02-20T22:46:04.892Z 0 000-019 100 100 100 176 176 176 F F F
109262024-02-20T22:46:04.892Z 1 020-039 100 100 100 176 176 176 F F F
109272024-02-20T22:46:04.892Z 2 040-059 100 100 100 175 175 175 F F F
109282024-02-20T22:46:04.892Z 3 060-079 99 99 99 174 174 174 F F F
109292024-02-20T22:46:04.892Z 4 080-099 97 97 97 170 170 170 F F F
109302024-02-20T22:46:04.892Z 5 100-119 100 100 100 176 176 176 F F F
109312024-02-20T22:46:04.892Z 6 120-139 100 100 100 176 176 176 F F F
109322024-02-20T22:46:04.892Z 7 140-159 95 95 95 168 168 168 F F F
109332024-02-20T22:46:04.892Z 8 160-179 94 94 94 167 167 167 F F F
109342024-02-20T22:46:04.892Z 9 180-199 94 94 94 167 167 167 F F F
109352024-02-20T22:46:04.897Z 10 200-219 97 97 97 170 170 170 F F F
109362024-02-20T22:46:04.897Z 11 220-239 99 99 99 174 174 174 F F F
109372024-02-20T22:46:04.897Z 12 240-259 100 100 100 176 176 176 F F F
109382024-02-20T22:46:04.897Z 13 260-279 101 101 101 177 177 177 F F F
109392024-02-20T22:46:04.897Z 14 280-299 92 92 92 163 163 163 F F F
109402024-02-20T22:46:04.897Z 15 300-319 99 99 99 174 174 174 F F F
109412024-02-20T22:46:04.897Z 16 320-339 100 100 100 176 176 176 F F F
109422024-02-20T22:46:04.897Z 17 340-359 100 100 100 176 176 176 F F F
109432024-02-20T22:46:04.897Z 18 360-379 94 94 94 166 166 166 F F F
109442024-02-20T22:46:04.897Z 19 380-399 92 92 92 162 162 162 F F F
109452024-02-20T22:46:04.897Z 20 400-419 101 101 101 177 177 177 F F F
109462024-02-20T22:46:04.897Z 21 420-439 101 101 101 177 177 177 F F F
109472024-02-20T22:46:04.897Z 22 440-459 98 98 98 173 173 173 F F F
109482024-02-20T22:46:04.897Z 23 460-479 98 98 98 173 173 173 F F F
109492024-02-20T22:46:04.897Z 24 480-499 99 99 99 174 174 174 F F F
109502024-02-20T22:46:04.897Z 25 500-519 100 100 100 176 176 176 F F F
109512024-02-20T22:46:04.897Z 26 520-539 100 100 100 176 176 176 F F F
109522024-02-20T22:46:04.897Z 27 540-559 100 100 100 175 175 175 F F F
109532024-02-20T22:46:04.898Z 28 560-579 100 100 100 175 175 175 F F F
109542024-02-20T22:46:04.898Z 29 580-599 101 101 101 177 177 177 F F F
109552024-02-20T22:46:04.898ZMax gen: 101, Max flush: 177
109562024-02-20T22:46:04.898Z
109572024-02-20T22:46:04.898ZBegin loop 51
109582024-02-20T22:46:04.898ZDownstairs to restart: 0
109592024-02-20T22:46:04.911Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432032433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546}
109602024-02-20T22:46:04.911Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432345952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546}
109612024-02-20T22:46:04.911Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432371301Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546}
109622024-02-20T22:46:04.911ZCrucible runtime is spawned
109632024-02-20T22:46:04.918Z{"msg":"Crucible 47cf1f43-5493-4d8c-ab5d-bbfa8c3931db has session id: 012569d8-23ec-425f-b7f0-4f7c42e8b0c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.43241279Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109642024-02-20T22:46:04.918Z{"msg":"Upstairs opts: Upstairs UUID: 47cf1f43-5493-4d8c-ab5d-bbfa8c3931db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432462648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109652024-02-20T22:46:04.918Z{"msg":"Crucible stats registered with UUID: 47cf1f43-5493-4d8c-ab5d-bbfa8c3931db","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432485357Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109662024-02-20T22:46:04.918Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432527376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546}
109672024-02-20T22:46:04.919Z{"msg":"47cf1f43-5493-4d8c-ab5d-bbfa8c3931db active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432645991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109682024-02-20T22:46:04.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109692024-02-20T22:46:04.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432732568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109702024-02-20T22:46:04.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432751438Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109712024-02-20T22:46:04.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432769197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109722024-02-20T22:46:04.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432803716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109732024-02-20T22:46:04.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432829155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109742024-02-20T22:46:04.919Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432856254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109752024-02-20T22:46:04.919Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432889603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109762024-02-20T22:46:04.919Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.432920342Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109772024-02-20T22:46:04.919Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433027598Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109782024-02-20T22:46:04.919Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433166963Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109792024-02-20T22:46:04.919Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433209941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109802024-02-20T22:46:04.919Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433683014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109812024-02-20T22:46:04.919Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433729233Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109822024-02-20T22:46:04.919Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433757082Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109832024-02-20T22:46:04.919Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.43379943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109842024-02-20T22:46:04.919Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433861028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109852024-02-20T22:46:04.919Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.433908816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109862024-02-20T22:46:04.919Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434270003Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109872024-02-20T22:46:04.919Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434306632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109882024-02-20T22:46:04.919Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434565233Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109892024-02-20T22:46:04.919Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434779455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109902024-02-20T22:46:04.919Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434815114Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109912024-02-20T22:46:04.919Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434830803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109922024-02-20T22:46:04.920Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434860952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109932024-02-20T22:46:04.920Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434883941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109942024-02-20T22:46:04.920Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.434898391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109952024-02-20T22:46:04.920Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435005037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109962024-02-20T22:46:04.920Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435047295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109972024-02-20T22:46:04.920Z{"msg":"[0]R flush_numbers[0..12]: [176, 176, 175, 174, 170, 176, 176, 168, 167, 167, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435070345Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109982024-02-20T22:46:04.920Z{"msg":"[0]R generation[0..12]: [100, 100, 100, 99, 97, 100, 100, 95, 94, 94, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435093914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
109992024-02-20T22:46:04.920Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435108933Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110002024-02-20T22:46:04.920Z{"msg":"[1]R flush_numbers[0..12]: [176, 176, 175, 174, 170, 176, 176, 168, 167, 167, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435129792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110012024-02-20T22:46:04.920Z{"msg":"[1]R generation[0..12]: [100, 100, 100, 99, 97, 100, 100, 95, 94, 94, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435144222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110022024-02-20T22:46:04.920Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435165871Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110032024-02-20T22:46:04.920Z{"msg":"[2]R flush_numbers[0..12]: [176, 176, 175, 174, 170, 176, 176, 168, 167, 167, 170, 174]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
110042024-02-20T22:46:04.920Z,"time":"2024-02-20T22:46:04.435180941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110052024-02-20T22:46:04.920Z{"msg":"[2]R generation[0..12]: [100, 100, 100, 99, 97, 100, 100, 95, 94, 94, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.43519861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110062024-02-20T22:46:04.920Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435220299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110072024-02-20T22:46:04.920Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435234549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110082024-02-20T22:46:04.920Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435248228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110092024-02-20T22:46:04.920Z{"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435268778Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110102024-02-20T22:46:04.920Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435282367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110112024-02-20T22:46:04.920Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435295746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110122024-02-20T22:46:04.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435320256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110132024-02-20T22:46:04.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435343125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110142024-02-20T22:46:04.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435361894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110152024-02-20T22:46:04.920Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435379174Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110162024-02-20T22:46:04.920Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435394633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110172024-02-20T22:46:04.921Z{"msg":"47cf1f43-5493-4d8c-ab5d-bbfa8c3931db is now active with session: 012569d8-23ec-425f-b7f0-4f7c42e8b0c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435418412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110182024-02-20T22:46:04.921Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.435446881Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546,"session_id":"012569d8-23ec-425f-b7f0-4f7c42e8b0c9"}
110192024-02-20T22:46:04.921Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:04.43546913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546}
110202024-02-20T22:46:04.921ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110212024-02-20T22:46:04.921ZLoading write count information from file "/tmp/test_repair_verify.data"
110222024-02-20T22:46:04.921ZRun Repair workload
110232024-02-20T22:46:04.921Z01/30 Write block 2 len 12288 data: 7 8 8
110242024-02-20T22:46:04.925Z02/30 Write block 461 len 4096 data: 13
110252024-02-20T22:46:04.925Z03/30 Read block 451 len 4096
110262024-02-20T22:46:04.925Z04/30 Write block 267 len 4096 data: 10
110272024-02-20T22:46:04.925Z05/30 Write block 535 len 4096 data: 11
110282024-02-20T22:46:04.925Z06/30 Read block 355 len 16384
110292024-02-20T22:46:04.925Z07/30 Write block 521 len 12288 data: 8 7 4
110302024-02-20T22:46:04.925Z08/30 Read block 308 len 4096
110312024-02-20T22:46:04.925Z09/30 Read block 267 len 20480
110322024-02-20T22:46:04.925Z10/30 Read block 98 len 20480
110332024-02-20T22:46:04.925Z11/30 Read block 222 len 32768
110342024-02-20T22:46:04.926Z12/30 Read block 208 len 8192
110352024-02-20T22:46:04.926Z13/30 Write block 186 len 8192 data: 7 8
110362024-02-20T22:46:04.926Z14/30 Read block 0 len 40960
110372024-02-20T22:46:04.926Z15/30 Write block 411 len 16384 data: 12 12 12 12
110382024-02-20T22:46:04.930Z16/30 Write block 538 len 28672 data: 11 9 11 10 8 9 8
110392024-02-20T22:46:04.930Z17/30 Write block 296 len 40960 data: 5 6 8 8 7 11 13 13 13 12
110402024-02-20T22:46:04.930Z18/30 Read block 302 len 40960
110412024-02-20T22:46:04.930Z19/30 Write block 177 len 32768 data: 10 10 11 11 11 11 8 9
110422024-02-20T22:46:04.930Z20/30 Read block 118 len 32768
110432024-02-20T22:46:04.934Z21/30 Read block 270 len 36864
110442024-02-20T22:46:04.934Z22/30 Read block 505 len 24576
110452024-02-20T22:46:04.934Z23/30 Write block 399 len 12288 data: 9 9 7
110462024-02-20T22:46:04.934Z24/30 Read block 35 len 28672
110472024-02-20T22:46:04.937Z25/30 Read block 346 len 32768
110482024-02-20T22:46:04.938Z26/30 Write block 157 len 32768 data: 9 8 8 8 6 5 3 3
110492024-02-20T22:46:04.938Z27/30 Read block 535 len 16384
110502024-02-20T22:46:04.943Z28/30 Read block 36 len 24576
110512024-02-20T22:46:04.943Z29/30 Read block 42 len 20480
110522024-02-20T22:46:04.943Z30/30 Write block 332 len 40960 data: 11 13 11 9 10 11 11 12 11 12
110532024-02-20T22:46:04.943Z----------------------------------------------------------------
110542024-02-20T22:46:04.943Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110552024-02-20T22:46:04.943ZStates: Active Active Active
110562024-02-20T22:46:04.943ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110572024-02-20T22:46:04.943Z 1 Acked 1000 Write 3 Done Done Done false
110582024-02-20T22:46:04.943Z 2 Acked 1001 Write 1 Sent Done Done false
110592024-02-20T22:46:04.944Z 3 Acked 1002 Read 1 Sent Done Done false
110602024-02-20T22:46:04.944Z 4 Acked 1003 Write 1 Sent Done Done false
110612024-02-20T22:46:04.944Z 5 Acked 1004 Write 1 Sent Done Done false
110622024-02-20T22:46:04.944Z 6 Acked 1005 Read 4 Sent Done Done false
110632024-02-20T22:46:04.944Z 7 Acked 1006 Write 3 Sent Done Done false
110642024-02-20T22:46:04.944Z 8 Acked 1007 Read 1 Sent Done Done false
110652024-02-20T22:46:04.944Z 9 Acked 1008 Read 5 Sent Done Done false
110662024-02-20T22:46:04.944Z 10 Acked 1009 Read 5 Sent Done Done false
110672024-02-20T22:46:04.944Z 11 Acked 1010 Read 8 Sent Done Done false
110682024-02-20T22:46:04.944Z 12 Acked 1011 Read 2 Sent Done Done false
110692024-02-20T22:46:04.944Z 13 Acked 1012 Write 2 Sent Done Done false
110702024-02-20T22:46:04.944Z 14 Acked 1013 Read 10 Sent Done Done false
110712024-02-20T22:46:04.944Z 15 Acked 1014 Write 4 Sent Done Done false
110722024-02-20T22:46:04.944Z 16 Acked 1015 Write 7 Sent Done Done false
110732024-02-20T22:46:04.944Z 17 Acked 1016 Write 10 Sent Done Done false
110742024-02-20T22:46:04.944Z 18 Acked 1017 Read 10 Sent Done Done false
110752024-02-20T22:46:04.944Z 19 Acked 1018 Write 8 Sent Done Done false
110762024-02-20T22:46:04.944Z 20 Acked 1019 Read 8 Sent Done Done false
110772024-02-20T22:46:04.944Z 21 Acked 1020 Read 9 Sent Done Done false
110782024-02-20T22:46:04.944Z 22 Acked 1021 Read 6 Sent Done Done false
110792024-02-20T22:46:04.944Z 23 Acked 1022 Write 3 Sent Done Done false
110802024-02-20T22:46:04.944Z 24 Acked 1023 Read 7 Sent Done Done false
110812024-02-20T22:46:04.944Z 25 Acked 1024 Read 8 Sent Done Done false
110822024-02-20T22:46:04.944Z 26 Acked 1025 Write 8 Sent Done Done false
110832024-02-20T22:46:04.944Z 27 Acked 1026 Read 4 Sent Done Done false
110842024-02-20T22:46:04.945Z 28 Acked 1027 Read 6 Sent Done Done false
110852024-02-20T22:46:04.945Z 29 Acked 1028 Read 5 Sent Done Done false
110862024-02-20T22:46:04.945Z 30 Acked 1029 Write 10 Sent Sent Sent false
110872024-02-20T22:46:04.945Z STATES DS:0 DS:1 DS:2 TOTAL
110882024-02-20T22:46:04.945Z New 0 0 0 0
110892024-02-20T22:46:04.945Z Sent 29 1 1 31
110902024-02-20T22:46:04.945Z Done 1 29 29 59
110912024-02-20T22:46:04.945Z Skipped 0 0 0 0
110922024-02-20T22:46:04.945Z Error 0 0 0 0
110932024-02-20T22:46:04.945ZLast Flush: 0 0 0
110942024-02-20T22:46:04.945ZDownstairs last five completed:
110952024-02-20T22:46:04.945ZUpstairs last five completed: 30 29 28 27 26
110962024-02-20T22:46:04.945Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:04.462408536Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1546}
110972024-02-20T22:46:04.945ZWrote out file "/tmp/test_repair_verify.data"
110982024-02-20T22:46:04.949Z
110992024-02-20T22:46:04.949ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111002024-02-20T22:46:04.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111012024-02-20T22:46:04.968Z 0 000-019 100 100 100 176 176 176 T T T
111022024-02-20T22:46:04.968Z 1 020-039 100 100 100 176 176 176 F F F
111032024-02-20T22:46:04.968Z 2 040-059 100 100 100 175 175 175 F F F
111042024-02-20T22:46:04.968Z 3 060-079 99 99 99 174 174 174 F F F
111052024-02-20T22:46:04.968Z 4 080-099 97 97 97 170 170 170 F F F
111062024-02-20T22:46:04.968Z 5 100-119 100 100 100 176 176 176 F F F
111072024-02-20T22:46:04.968Z 6 120-139 100 100 100 176 176 176 F F F
111082024-02-20T22:46:04.968Z 7 140-159 95 95 95 168 168 168 F T T <---
111092024-02-20T22:46:04.968Z 8 160-179 94 94 94 167 167 167 F T T <---
111102024-02-20T22:46:04.969Z 9 180-199 94 94 94 167 167 167 F T T <---
111112024-02-20T22:46:04.969Z 10 200-219 97 97 97 170 170 170 F F F
111122024-02-20T22:46:04.969Z 11 220-239 99 99 99 174 174 174 F F F
111132024-02-20T22:46:04.969Z 12 240-259 100 100 100 176 176 176 F F F
111142024-02-20T22:46:04.969Z 13 260-279 101 101 101 177 177 177 F T T <---
111152024-02-20T22:46:04.969Z 14 280-299 92 92 92 163 163 163 F T T <---
111162024-02-20T22:46:04.974Z 15 300-319 99 99 99 174 174 174 F T T <---
111172024-02-20T22:46:04.974Z 16 320-339 100 100 100 176 176 176 F T T <---
111182024-02-20T22:46:04.974Z 17 340-359 100 100 100 176 176 176 F T T <---
111192024-02-20T22:46:04.974Z 18 360-379 94 94 94 166 166 166 F F F
111202024-02-20T22:46:04.974Z 19 380-399 92 92 92 162 162 162 F T T <---
111212024-02-20T22:46:04.974Z 20 400-419 101 101 101 177 177 177 F T T <---
111222024-02-20T22:46:04.974Z 21 420-439 101 101 101 177 177 177 F F F
111232024-02-20T22:46:04.974Z 22 440-459 98 98 98 173 173 173 F F F
111242024-02-20T22:46:04.974Z 23 460-479 98 98 98 173 173 173 F T T <---
111252024-02-20T22:46:04.974Z 24 480-499 99 99 99 174 174 174 F F F
111262024-02-20T22:46:04.974Z 25 500-519 100 100 100 176 176 176 F F F
111272024-02-20T22:46:04.974Z 26 520-539 100 100 100 176 176 176 F T T <---
111282024-02-20T22:46:04.974Z 27 540-559 100 100 100 175 175 175 F T T <---
111292024-02-20T22:46:04.974Z 28 560-579 100 100 100 175 175 175 F F F
111302024-02-20T22:46:04.975Z 29 580-599 101 101 101 177 177 177 F F F
111312024-02-20T22:46:04.975ZMax gen: 101, Max flush: 177
111322024-02-20T22:46:04.975ZError: Difference in extent metadata found!
111332024-02-20T22:46:04.975ZA Difference in extent metadata is expected here
111342024-02-20T22:46:04.975ZOn loop 51
111352024-02-20T22:46:04.975Z
111362024-02-20T22:46:04.979ZVerifying data now
111372024-02-20T22:46:09.665ZNo change after verify
111382024-02-20T22:46:09.665ZLoop: 51 Downstairs dump after verify (and repair):
111392024-02-20T22:46:09.679ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111402024-02-20T22:46:09.679Z 0 000-019 102 102 102 178 178 178 F F F
111412024-02-20T22:46:09.679Z 1 020-039 100 100 100 176 176 176 F F F
111422024-02-20T22:46:09.679Z 2 040-059 100 100 100 175 175 175 F F F
111432024-02-20T22:46:09.679Z 3 060-079 99 99 99 174 174 174 F F F
111442024-02-20T22:46:09.679Z 4 080-099 97 97 97 170 170 170 F F F
111452024-02-20T22:46:09.679Z 5 100-119 100 100 100 176 176 176 F F F
111462024-02-20T22:46:09.679Z 6 120-139 100 100 100 176 176 176 F F F
111472024-02-20T22:46:09.679Z 7 140-159 102 102 102 178 178 178 F F F
111482024-02-20T22:46:09.683Z 8 160-179 102 102 102 178 178 178 F F F
111492024-02-20T22:46:09.684Z 9 180-199 102 102 102 178 178 178 F F F
111502024-02-20T22:46:09.684Z 10 200-219 97 97 97 170 170 170 F F F
111512024-02-20T22:46:09.684Z 11 220-239 99 99 99 174 174 174 F F F
111522024-02-20T22:46:09.684Z 12 240-259 100 100 100 176 176 176 F F F
111532024-02-20T22:46:09.684Z 13 260-279 102 102 102 178 178 178 F F F
111542024-02-20T22:46:09.684Z 14 280-299 102 102 102 178 178 178 F F F
111552024-02-20T22:46:09.684Z 15 300-319 102 102 102 178 178 178 F F F
111562024-02-20T22:46:09.684Z 16 320-339 102 102 102 178 178 178 F F F
111572024-02-20T22:46:09.684Z 17 340-359 102 102 102 178 178 178 F F F
111582024-02-20T22:46:09.684Z 18 360-379 94 94 94 166 166 166 F F F
111592024-02-20T22:46:09.684Z 19 380-399 102 102 102 178 178 178 F F F
111602024-02-20T22:46:09.684Z 20 400-419 102 102 102 178 178 178 F F F
111612024-02-20T22:46:09.684Z 21 420-439 101 101 101 177 177 177 F F F
111622024-02-20T22:46:09.684Z 22 440-459 98 98 98 173 173 173 F F F
111632024-02-20T22:46:09.684Z 23 460-479 102 102 102 178 178 178 F F F
111642024-02-20T22:46:09.684Z 24 480-499 99 99 99 174 174 174 F F F
111652024-02-20T22:46:09.684Z 25 500-519 100 100 100 176 176 176 F F F
111662024-02-20T22:46:09.684Z 26 520-539 102 102 102 178 178 178 F F F
111672024-02-20T22:46:09.684Z 27 540-559 102 102 102 178 178 178 F F F
111682024-02-20T22:46:09.684Z 28 560-579 100 100 100 175 175 175 F F F
111692024-02-20T22:46:09.684Z 29 580-599 101 101 101 177 177 177 F F F
111702024-02-20T22:46:09.684ZMax gen: 102, Max flush: 178
111712024-02-20T22:46:09.685Z
111722024-02-20T22:46:09.685ZBegin loop 52
111732024-02-20T22:46:09.685ZDownstairs to restart: 2
111742024-02-20T22:46:09.702Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.222723778Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554}
111752024-02-20T22:46:09.708Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223082495Z","hostname":"Crucible runtime is spawned
111762024-02-20T22:46:09.708Zip-10-150-1-191.us-west-2.compute.internal","pid":1554}
111772024-02-20T22:46:09.708Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223134023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554}
111782024-02-20T22:46:09.708Z{"msg":"Crucible 3a589762-8089-41ae-870c-d4f34a73fcdd has session id: 37c9a82e-397e-4e89-81c5-0010f3bbb6f8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223160512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111792024-02-20T22:46:09.708Z{"msg":"Upstairs opts: Upstairs UUID: 3a589762-8089-41ae-870c-d4f34a73fcdd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223202331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111802024-02-20T22:46:09.708Z{"msg":"Crucible stats registered with UUID: 3a589762-8089-41ae-870c-d4f34a73fcdd","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22321987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111812024-02-20T22:46:09.708Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22324049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554}
111822024-02-20T22:46:09.708Z{"msg":"3a589762-8089-41ae-870c-d4f34a73fcdd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223326776Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111832024-02-20T22:46:09.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223365625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111842024-02-20T22:46:09.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223392904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111852024-02-20T22:46:09.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223433872Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111862024-02-20T22:46:09.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223463231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111872024-02-20T22:46:09.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22350002Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111882024-02-20T22:46:09.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223537039Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111892024-02-20T22:46:09.709Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223563018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111902024-02-20T22:46:09.709Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223605866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111912024-02-20T22:46:09.709Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223650545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111922024-02-20T22:46:09.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223684594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111932024-02-20T22:46:09.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223724842Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111942024-02-20T22:46:09.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.223763451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111952024-02-20T22:46:09.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.224438527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111962024-02-20T22:46:09.709Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.224475165Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111972024-02-20T22:46:09.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.224494905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111982024-02-20T22:46:09.709Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.224531723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
111992024-02-20T22:46:09.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.224551483Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112002024-02-20T22:46:09.709Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.224574402Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112012024-02-20T22:46:09.709Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225023586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112022024-02-20T22:46:09.709Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225076414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112032024-02-20T22:46:09.709Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225336215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112042024-02-20T22:46:09.709Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225559607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112052024-02-20T22:46:09.709Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225618834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112062024-02-20T22:46:09.709Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225664423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112072024-02-20T22:46:09.709Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225701651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112082024-02-20T22:46:09.709Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22574004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112092024-02-20T22:46:09.709Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225768149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112102024-02-20T22:46:09.709Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225840906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112112024-02-20T22:46:09.709Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225880115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112122024-02-20T22:46:09.710Z{"msg":"[0]R flush_numbers[0..12]: [178, 176, 175, 174, 170, 176, 176, 178, 178, 178, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225924434Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112132024-02-20T22:46:09.710Z{"msg":"[0]R generation[0..12]: [102, 100, 100, 99, 97, 100, 100, 102, 102, 102, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225960932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112142024-02-20T22:46:09.710Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.225978901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112152024-02-20T22:46:09.710Z{"msg":"[1]R flush_numbers[0..12]: [178, 176, 175, 174, 170, 176, 176, 178, 178, 178, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226001301Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112162024-02-20T22:46:09.710Z{"msg":"[1]R generation[0..12]: [102, 100, 100, 99, 97, 100, 100, 102, 102, 102, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22601638Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112172024-02-20T22:46:09.710ZWait for a query_work_queue command to finish before sending IO
112182024-02-20T22:46:09.710Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226043989Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112192024-02-20T22:46:09.710Z{"msg":"[2]R flush_numbers[0..12]: [178, 176, 175, 174, 170, 176, 176, 178, 178, 178, 170, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226059718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112202024-02-20T22:46:09.710Z{"msg":"[2]R generation[0..12]: [102, 100, 100, 99, 97, 100, 100, 102, 102, 102, 97, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226074328Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112212024-02-20T22:46:09.710Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226095447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112222024-02-20T22:46:09.710Z{"msg":"Max found gen is 103","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226109177Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112232024-02-20T22:46:09.710Z{"msg":"Generation requested: 104 >= found:103","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226130016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112242024-02-20T22:46:09.710Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226149315Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112252024-02-20T22:46:09.710Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226174945Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112262024-02-20T22:46:09.710Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226189574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112272024-02-20T22:46:09.710Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226201784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112282024-02-20T22:46:09.710Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226222373Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112292024-02-20T22:46:09.710Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226236072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112302024-02-20T22:46:09.710Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226255911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112312024-02-20T22:46:09.710Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226268801Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112322024-02-20T22:46:09.710Z{"msg":"3a589762-8089-41ae-870c-d4f34a73fcdd is now active with session: 37c9a82e-397e-4e89-81c5-0010f3bbb6f8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.226282041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112332024-02-20T22:46:09.710Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22629489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554,"session_id":"37c9a82e-397e-4e89-81c5-0010f3bbb6f8"}
112342024-02-20T22:46:09.710Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:09.22630763Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554}
112352024-02-20T22:46:09.710ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112362024-02-20T22:46:09.710ZLoading write count information from file "/tmp/test_repair_verify.data"
112372024-02-20T22:46:09.711ZRun Repair workload
112382024-02-20T22:46:09.714Z01/30 Write block 83 len 32768 data: 5 5 4 4 5 5 6 7
112392024-02-20T22:46:09.714Z02/30 Read block 65 len 32768
112402024-02-20T22:46:09.715Z03/30 Read block 358 len 16384
112412024-02-20T22:46:09.715Z04/30 Write block 254 len 36864 data: 6 4 4 5 7 6 6 6 6
112422024-02-20T22:46:09.715Z05/30 Read block 465 len 16384
112432024-02-20T22:46:09.715Z06/30 Flush
112442024-02-20T22:46:09.715Z07/30 Write block 144 len 36864 data: 9 8 6 5 7 9 12 13 11
112452024-02-20T22:46:09.715Z08/30 Read block 64 len 32768
112462024-02-20T22:46:09.724Z09/30 Read block 71 len 36864
112472024-02-20T22:46:09.724Z10/30 Read block 19 len 8192
112482024-02-20T22:46:09.724Z11/30 Read block 512 len 32768
112492024-02-20T22:46:09.724Z12/30 Read block 103 len 36864
112502024-02-20T22:46:09.724Z13/30 Read block 347 len 12288
112512024-02-20T22:46:09.724Z14/30 Flush
112522024-02-20T22:46:09.728Z15/30 Write block 228 len 12288 data: 8 11 14
112532024-02-20T22:46:09.728Z16/30 Read block 492 len 36864
112542024-02-20T22:46:09.728Z17/30 Write block 596 len 12288 data: 11 10 8
112552024-02-20T22:46:09.728Z18/30 Write block 287 len 8192 data: 12 11
112562024-02-20T22:46:09.728Z19/30 Read block 131 len 36864
112572024-02-20T22:46:09.728Z20/30 Write block 163 len 16384 data: 4 4 6 7
112582024-02-20T22:46:09.729Z21/30 Write block 63 len 20480 data: 4 4 5 5 9
112592024-02-20T22:46:09.729Z22/30 Read block 262 len 8192
112602024-02-20T22:46:09.733Z23/30 Write block 338 len 12288 data: 12 13 12
112612024-02-20T22:46:09.734Z24/30 Read block 429 len 8192
112622024-02-20T22:46:09.734Z25/30 Write block 378 len 20480 data: 9 6 6 5 7
112632024-02-20T22:46:09.734Z26/30 Read block 543 len 32768
112642024-02-20T22:46:09.734Z27/30 Read block 174 len 20480
112652024-02-20T22:46:09.739Z28/30 Read block 142 len 4096
112662024-02-20T22:46:09.739Z29/30 Read block 84 len 12288
112672024-02-20T22:46:09.740Z30/30 Write block 578 len 12288 data: 11 13 14
112682024-02-20T22:46:09.740Z----------------------------------------------------------------
112692024-02-20T22:46:09.740Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112702024-02-20T22:46:09.740ZStates: Active Active Active
112712024-02-20T22:46:09.740ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112722024-02-20T22:46:09.740Z 1 Acked 1000 Write 8 Done Done Done false
112732024-02-20T22:46:09.740Z 2 Acked 1001 Read 8 Done Done Done false
112742024-02-20T22:46:09.740Z 3 Acked 1002 Read 4 Done Done Sent false
112752024-02-20T22:46:09.740Z 4 Acked 1003 Write 9 Done Done Sent false
112762024-02-20T22:46:09.740Z 5 Acked 1004 Read 4 Done Done Sent false
112772024-02-20T22:46:09.740Z 6 Acked 1005 Flush 0 Done Done Sent false
112782024-02-20T22:46:09.740Z 7 Acked 1006 Write 9 Done Done Sent false
112792024-02-20T22:46:09.740Z 8 Acked 1007 Read 8 Done Done Sent false
112802024-02-20T22:46:09.740Z 9 Acked 1008 Read 9 Done Done Sent false
112812024-02-20T22:46:09.740Z 10 Acked 1009 Read 2 Done Done Sent false
112822024-02-20T22:46:09.740Z 11 Acked 1010 Read 8 Done Done Sent false
112832024-02-20T22:46:09.740Z 12 Acked 1011 Read 9 Done Done Sent false
112842024-02-20T22:46:09.740Z 13 Acked 1012 Read 3 Done Done Sent false
112852024-02-20T22:46:09.740Z 14 Acked 1013 Flush 0 Done Done Sent false
112862024-02-20T22:46:09.740Z 15 Acked 1014 Write 3 Done Done Sent false
112872024-02-20T22:46:09.740Z 16 Acked 1015 Read 9 Done Done Sent false
112882024-02-20T22:46:09.740Z 17 Acked 1016 Write 3 Done Done Sent false
112892024-02-20T22:46:09.740Z 18 Acked 1017 Write 2 Done Done Sent false
112902024-02-20T22:46:09.740Z 19 Acked 1018 Read 9 Done Done Sent false
112912024-02-20T22:46:09.740Z 20 Acked 1019 Write 4 Done Done Sent false
112922024-02-20T22:46:09.741Z 21 Acked 1020 Write 5 Done Done Sent false
112932024-02-20T22:46:09.741Z 22 Acked 1021 Read 2 Done Done Sent false
112942024-02-20T22:46:09.741Z 23 Acked 1022 Write 3 Done Done Sent false
112952024-02-20T22:46:09.741Z 24 Acked 1023 Read 2 Done Done Sent false
112962024-02-20T22:46:09.741Z 25 Acked 1024 Write 5 Done Done Sent false
112972024-02-20T22:46:09.741Z 26 Acked 1025 Read 8 Done Done Sent false
112982024-02-20T22:46:09.741Z 27 Acked 1026 Read 5 Done Done Sent false
112992024-02-20T22:46:09.741Z 28 Acked 1027 Read 1 Done Done Sent false
113002024-02-20T22:46:09.741Z 29 Acked 1028 Read 3 Done Done Sent false
113012024-02-20T22:46:09.741Z 30 Acked 1029 Write 3 Sent Sent Sent false
113022024-02-20T22:46:09.741Z STATES DS:0 DS:1 DS:2 TOTAL
113032024-02-20T22:46:09.741Z New 0 0 0 0
113042024-02-20T22:46:09.741Z Sent 1 1 28 30
113052024-02-20T22:46:09.741Z Done 29 29 2 60
113062024-02-20T22:46:09.741Z Skipped 0 0 0 0
113072024-02-20T22:46:09.741Z Error 0 0 0 0
113082024-02-20T22:46:09.741ZLast Flush: 1013 1013 0
113092024-02-20T22:46:09.741ZDownstairs last five completed:
113102024-02-20T22:46:09.741ZUpstairs last five completed: 30 29 28 27 26
113112024-02-20T22:46:09.741Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:09.256625975Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1554}
113122024-02-20T22:46:09.741ZWrote out file "/tmp/test_repair_verify.data"
113132024-02-20T22:46:09.753Z
113142024-02-20T22:46:09.753ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113152024-02-20T22:46:09.761ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113162024-02-20T22:46:09.761Z 0 000-019 102 102 102 178 178 178 F F F
113172024-02-20T22:46:09.761Z 1 020-039 100 100 100 176 176 176 F F F
113182024-02-20T22:46:09.761Z 2 040-059 100 100 100 175 175 175 F F F
113192024-02-20T22:46:09.761Z 3 060-079 99 99 99 174 174 174 T T F <---
113202024-02-20T22:46:09.761Z 4 080-099 104 104 97 179 179 170 F F T <---
113212024-02-20T22:46:09.768Z 5 100-119 100 100 100 176 176 176 F F F
113222024-02-20T22:46:09.768Z 6 120-139 100 100 100 176 176 176 F F F
113232024-02-20T22:46:09.768Z 7 140-159 104 104 102 180 180 178 F F F <---
113242024-02-20T22:46:09.768Z 8 160-179 102 102 102 178 178 178 T T F <---
113252024-02-20T22:46:09.768Z 9 180-199 102 102 102 178 178 178 F F F
113262024-02-20T22:46:09.768Z 10 200-219 97 97 97 170 170 170 F F F
113272024-02-20T22:46:09.768Z 11 220-239 99 99 99 174 174 174 T T F <---
113282024-02-20T22:46:09.768Z 12 240-259 104 104 100 179 179 176 F F F <---
113292024-02-20T22:46:09.768Z 13 260-279 104 104 102 179 179 178 F F F <---
113302024-02-20T22:46:09.768Z 14 280-299 102 102 102 178 178 178 T T F <---
113312024-02-20T22:46:09.768Z 15 300-319 102 102 102 178 178 178 F F F
113322024-02-20T22:46:09.768Z 16 320-339 102 102 102 178 178 178 T T F <---
113332024-02-20T22:46:09.768Z 17 340-359 102 102 102 178 178 178 T T F <---
113342024-02-20T22:46:09.768Z 18 360-379 94 94 94 166 166 166 T T F <---
113352024-02-20T22:46:09.768Z 19 380-399 102 102 102 178 178 178 T T F <---
113362024-02-20T22:46:09.769Z 20 400-419 102 102 102 178 178 178 F F F
113372024-02-20T22:46:09.769Z 21 420-439 101 101 101 177 177 177 F F F
113382024-02-20T22:46:09.769Z 22 440-459 98 98 98 173 173 173 F F F
113392024-02-20T22:46:09.769Z 23 460-479 102 102 102 178 178 178 F F F
113402024-02-20T22:46:09.769Z 24 480-499 99 99 99 174 174 174 F F F
113412024-02-20T22:46:09.769Z 25 500-519 100 100 100 176 176 176 F F F
113422024-02-20T22:46:09.769Z 26 520-539 102 102 102 178 178 178 F F F
113432024-02-20T22:46:09.769Z 27 540-559 102 102 102 178 178 178 F F F
113442024-02-20T22:46:09.769Z 28 560-579 100 100 100 175 175 175 T T F <---
113452024-02-20T22:46:09.769Z 29 580-599 101 101 101 177 177 177 T T F <---
113462024-02-20T22:46:09.769ZMax gen: 104, Max flush: 180
113472024-02-20T22:46:09.769ZError: Difference in extent metadata found!
113482024-02-20T22:46:09.769ZA Difference in extent metadata is expected here
113492024-02-20T22:46:09.769ZOn loop 52
113502024-02-20T22:46:09.769Z
113512024-02-20T22:46:09.776ZVerifying data now
113522024-02-20T22:46:14.440ZNo change after verify
113532024-02-20T22:46:14.440ZLoop: 52 Downstairs dump after verify (and repair):
113542024-02-20T22:46:14.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113552024-02-20T22:46:14.454Z 0 000-019 102 102 102 178 178 178 F F F
113562024-02-20T22:46:14.454Z 1 020-039 100 100 100 176 176 176 F F F
113572024-02-20T22:46:14.454Z 2 040-059 100 100 100 175 175 175 F F F
113582024-02-20T22:46:14.454Z 3 060-079 105 105 105 181 181 181 F F F
113592024-02-20T22:46:14.454Z 4 080-099 104 104 104 179 179 179 F F F
113602024-02-20T22:46:14.454Z 5 100-119 100 100 100 176 176 176 F F F
113612024-02-20T22:46:14.454Z 6 120-139 100 100 100 176 176 176 F F F
113622024-02-20T22:46:14.458Z 7 140-159 104 104 104 180 180 180 F F F
113632024-02-20T22:46:14.458Z 8 160-179 105 105 105 181 181 181 F F F
113642024-02-20T22:46:14.458Z 9 180-199 102 102 102 178 178 178 F F F
113652024-02-20T22:46:14.458Z 10 200-219 97 97 97 170 170 170 F F F
113662024-02-20T22:46:14.458Z 11 220-239 105 105 105 181 181 181 F F F
113672024-02-20T22:46:14.458Z 12 240-259 104 104 104 179 179 179 F F F
113682024-02-20T22:46:14.458Z 13 260-279 104 104 104 179 179 179 F F F
113692024-02-20T22:46:14.458Z 14 280-299 105 105 105 181 181 181 F F F
113702024-02-20T22:46:14.458Z 15 300-319 102 102 102 178 178 178 F F F
113712024-02-20T22:46:14.458Z 16 320-339 105 105 105 181 181 181 F F F
113722024-02-20T22:46:14.458Z 17 340-359 105 105 105 181 181 181 F F F
113732024-02-20T22:46:14.458Z 18 360-379 105 105 105 181 181 181 F F F
113742024-02-20T22:46:14.458Z 19 380-399 105 105 105 181 181 181 F F F
113752024-02-20T22:46:14.458Z 20 400-419 102 102 102 178 178 178 F F F
113762024-02-20T22:46:14.458Z 21 420-439 101 101 101 177 177 177 F F F
113772024-02-20T22:46:14.459Z 22 440-459 98 98 98 173 173 173 F F F
113782024-02-20T22:46:14.459Z 23 460-479 102 102 102 178 178 178 F F F
113792024-02-20T22:46:14.459Z 24 480-499 99 99 99 174 174 174 F F F
113802024-02-20T22:46:14.459Z 25 500-519 100 100 100 176 176 176 F F F
113812024-02-20T22:46:14.459Z 26 520-539 102 102 102 178 178 178 F F F
113822024-02-20T22:46:14.459Z 27 540-559 102 102 102 178 178 178 F F F
113832024-02-20T22:46:14.459Z 28 560-579 105 105 105 181 181 181 F F F
113842024-02-20T22:46:14.459Z 29 580-599 105 105 105 181 181 181 F F F
113852024-02-20T22:46:14.459ZMax gen: 105, Max flush: 181
113862024-02-20T22:46:14.459Z
113872024-02-20T22:46:14.459ZBegin loop 53
113882024-02-20T22:46:14.459ZDownstairs to restart: 0
113892024-02-20T22:46:14.476Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.996596614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562}
113902024-02-20T22:46:14.476Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.996878134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562}
113912024-02-20T22:46:14.482Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.996909793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562}
113922024-02-20T22:46:14.482Z{"msg":"Crucible 57f01b34-e4f5-47d3-a896-3761c5e14a05 has session id: d8fe1055-72db-4e72-b8c9-5fdb21672c16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.996938612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
113932024-02-20T22:46:14.482Z{"msg":"Upstairs opts: Upstairs UUID: 57f01b34-e4f5-47d3-a896-3761c5e14a05, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.99698151Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
113942024-02-20T22:46:14.482Z{"msg":"Crucible stats registered with UUID: 57f01b34-e4f5-47d3-a896-3761c5e14a05","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997033229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
113952024-02-20T22:46:14.482ZCrucible runtime is spawned
113962024-02-20T22:46:14.482Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997229231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562}
113972024-02-20T22:46:14.482Z{"msg":"57f01b34-e4f5-47d3-a896-3761c5e14a05 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997294129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
113982024-02-20T22:46:14.482Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997321158Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
113992024-02-20T22:46:14.482Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997359407Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114002024-02-20T22:46:14.482Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997388506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114012024-02-20T22:46:14.482Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997427184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114022024-02-20T22:46:14.482Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997455893Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114032024-02-20T22:46:14.482Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.9975417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114042024-02-20T22:46:14.482Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997580189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114052024-02-20T22:46:14.482Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997613588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114062024-02-20T22:46:14.482Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997647707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114072024-02-20T22:46:14.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997675555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114082024-02-20T22:46:14.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997849099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114092024-02-20T22:46:14.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.997914217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114102024-02-20T22:46:14.482Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.998415539Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114112024-02-20T22:46:14.482Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.998450888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114122024-02-20T22:46:14.483Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.998466697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114132024-02-20T22:46:14.483Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.998480447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114142024-02-20T22:46:14.483Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.998494606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114152024-02-20T22:46:14.483Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.998509876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114162024-02-20T22:46:14.483Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999039367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114172024-02-20T22:46:14.483Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999090015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114182024-02-20T22:46:14.483Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999132573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114192024-02-20T22:46:14.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999630685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114202024-02-20T22:46:14.483Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999675634Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114212024-02-20T22:46:14.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999693593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114222024-02-20T22:46:14.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999706753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114232024-02-20T22:46:14.483Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999728412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114242024-02-20T22:46:14.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999743052Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114252024-02-20T22:46:14.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999755761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114262024-02-20T22:46:14.483Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.99977686Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114272024-02-20T22:46:14.483Z{"msg":"[0]R flush_numbers[0..12]: [178, 176, 175, 181, 179, 176, 176, 180, 181, 178, 170, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.99979196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114282024-02-20T22:46:14.483Z{"msg":"[0]R generation[0..12]: [102, 100, 100, 105, 104, 100, 100, 104, 105, 102, 97, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999813079Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114292024-02-20T22:46:14.483Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999827578Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114302024-02-20T22:46:14.483Z{"msg":"[1]R flush_numbers[0..12]: [178, 176, 175, 181, 179, 176, 176, 180, 181, 178, 170, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999849938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114312024-02-20T22:46:14.483Z{"msg":"[1]R generation[0..12]: [102, 100, 100, 105, 104, 100, 100, 104, 105, 102, 97, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999865357Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114322024-02-20T22:46:14.483Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999890466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114332024-02-20T22:46:14.483Z{"msg":"[2]R flush_numbers[0..12]: [178, 176, 175, 181, 179, 176, 176, 180, 181, 178, 170, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999908815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114342024-02-20T22:46:14.483Z{"msg":"[2]R generation[0..12]: [102, 100, 100, 105, 104, 100, 100, 104, 105, 102, 97, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999922885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114352024-02-20T22:46:14.483Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999936865Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114362024-02-20T22:46:14.483ZWait for a query_work_queue command to finish before sending IO
114372024-02-20T22:46:14.484Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999967324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114382024-02-20T22:46:14.484Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:13.999982813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114392024-02-20T22:46:14.484Z{"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000291222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114402024-02-20T22:46:14.484Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000318851Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114412024-02-20T22:46:14.484Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.00033548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114422024-02-20T22:46:14.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.00034876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114432024-02-20T22:46:14.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000363829Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114442024-02-20T22:46:14.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000377789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114452024-02-20T22:46:14.484Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114462024-02-20T22:46:14.484Z,"time":"2024-02-20T22:46:14.000392098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114472024-02-20T22:46:14.484Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000407628Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114482024-02-20T22:46:14.484Z{"msg":"57f01b34-e4f5-47d3-a896-3761c5e14a05 is now active with session: d8fe1055-72db-4e72-b8c9-5fdb21672c16","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000422087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114492024-02-20T22:46:14.484Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000448866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562,"session_id":"d8fe1055-72db-4e72-b8c9-5fdb21672c16"}
114502024-02-20T22:46:14.484Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:14.000462596Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562}
114512024-02-20T22:46:14.488ZLoading write count information from file "/tmp/test_repair_verify.data"
114522024-02-20T22:46:14.488ZRun Repair workload
114532024-02-20T22:46:14.488Z01/30 Write block 520 len 20480 data: 8 9 8 5 3
114542024-02-20T22:46:14.488Z02/30 Write block 214 len 20480 data: 3 4 4 3 4
114552024-02-20T22:46:14.488Z03/30 Flush
114562024-02-20T22:46:14.488Z04/30 Write block 94 len 4096 data: 6
114572024-02-20T22:46:14.488Z05/30 Flush
114582024-02-20T22:46:14.488Z06/30 Write block 170 len 8192 data: 9 8
114592024-02-20T22:46:14.488Z07/30 Write block 377 len 28672 data: 12 10 7 7 6 8 6
114602024-02-20T22:46:14.488Z08/30 Read block 416 len 16384
114612024-02-20T22:46:14.492Z09/30 Read block 68 len 4096
114622024-02-20T22:46:14.492Z10/30 Read block 557 len 36864
114632024-02-20T22:46:14.492Z11/30 Write block 449 len 24576 data: 7 7 7 10 8 10
114642024-02-20T22:46:14.492Z12/30 Read block 558 len 36864
114652024-02-20T22:46:14.497Z13/30 Flush
114662024-02-20T22:46:14.497Z14/30 Write block 56 len 36864 data: 6 5 6 8 4 3 4 5 5
114672024-02-20T22:46:14.503Z15/30 Read block 286 len 36864
114682024-02-20T22:46:14.503Z16/30 Read block 16 len 28672
114692024-02-20T22:46:14.503Z17/30 Read block 136 len 16384
114702024-02-20T22:46:14.503Z18/30 Read block 138 len 20480
114712024-02-20T22:46:14.503Z19/30 Write block 132 len 4096 data: 9
114722024-02-20T22:46:14.503Z20/30 Flush
114732024-02-20T22:46:14.507Z21/30 Write block 18 len 32768 data: 10 10 10 7 8 8 7 12
114742024-02-20T22:46:14.507Z22/30 Read block 6 len 28672
114752024-02-20T22:46:14.507Z23/30 Read block 232 len 40960
114762024-02-20T22:46:14.507Z24/30 Write block 279 len 8192 data: 4 4
114772024-02-20T22:46:14.508Z25/30 Read block 102 len 24576
114782024-02-20T22:46:14.512Z26/30 Read block 490 len 4096
114792024-02-20T22:46:14.512Z27/30 Read block 388 len 24576
114802024-02-20T22:46:14.512Z28/30 Read block 166 len 36864
114812024-02-20T22:46:14.512Z29/30 Write block 172 len 40960 data: 9 9 9 11 10 11 11 12 12 12
114822024-02-20T22:46:14.512Z30/30 Read block 125 len 24576
114832024-02-20T22:46:14.517Z----------------------------------------------------------------
114842024-02-20T22:46:14.517Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114852024-02-20T22:46:14.518ZStates: Active Active Active
114862024-02-20T22:46:14.518ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114872024-02-20T22:46:14.518Z 1 Acked 1000 Write 5 Sent Done Done false
114882024-02-20T22:46:14.518Z 2 Acked 1001 Write 5 Sent Done Done false
114892024-02-20T22:46:14.518Z 3 Acked 1002 Flush 0 Sent Done Done false
114902024-02-20T22:46:14.518Z 4 Acked 1003 Write 1 Sent Done Done false
114912024-02-20T22:46:14.518Z 5 Acked 1004 Flush 0 Sent Done Done false
114922024-02-20T22:46:14.518Z 6 Acked 1005 Write 2 Sent Done Done false
114932024-02-20T22:46:14.518Z 7 Acked 1006 Write 7 Sent Done Done false
114942024-02-20T22:46:14.518Z 8 Acked 1007 Read 4 Sent Done Done false
114952024-02-20T22:46:14.518Z 9 Acked 1008 Read 1 Sent Done Done false
114962024-02-20T22:46:14.518Z 10 Acked 1009 Read 9 Sent Done Done false
114972024-02-20T22:46:14.518Z 11 Acked 1010 Write 6 Sent Done Done false
114982024-02-20T22:46:14.518Z 12 Acked 1011 Read 9 Sent Done Done false
114992024-02-20T22:46:14.518Z 13 Acked 1012 Flush 0 Sent Done Done false
115002024-02-20T22:46:14.518Z 14 Acked 1013 Write 9 Sent Done Done false
115012024-02-20T22:46:14.518Z 15 Acked 1014 Read 9 Sent Done Done false
115022024-02-20T22:46:14.518Z 16 Acked 1015 Read 7 Sent Done Done false
115032024-02-20T22:46:14.518Z 17 Acked 1016 Read 4 Sent Done Done false
115042024-02-20T22:46:14.518Z 18 Acked 1017 Read 5 Sent Done Done false
115052024-02-20T22:46:14.518Z 19 Acked 1018 Write 1 Sent Done Done false
115062024-02-20T22:46:14.518Z 20 Acked 1019 Flush 0 Sent Done Done false
115072024-02-20T22:46:14.518Z 21 Acked 1020 Write 8 Sent Done Done false
115082024-02-20T22:46:14.518Z 22 Acked 1021 Read 7 Sent Done Done false
115092024-02-20T22:46:14.519Z 23 Acked 1022 Read 10 Sent Done Done false
115102024-02-20T22:46:14.519Z 24 Acked 1023 Write 2 Sent Done Done false
115112024-02-20T22:46:14.519Z 25 Acked 1024 Read 6 Sent Done Done false
115122024-02-20T22:46:14.519Z 26 Acked 1025 Read 1 Sent Done Done false
115132024-02-20T22:46:14.519Z 27 Acked 1026 Read 6 Sent Done Done false
115142024-02-20T22:46:14.519Z 28 Acked 1027 Read 9 Sent Done Done false
115152024-02-20T22:46:14.519Z 29 Acked 1028 Write 10 Sent Done Done false
115162024-02-20T22:46:14.519Z 30 Acked 1029 Read 6 Sent Sent Done false
115172024-02-20T22:46:14.519Z STATES DS:0 DS:1 DS:2 TOTAL
115182024-02-20T22:46:14.519Z New 0 0 0 0
115192024-02-20T22:46:14.519Z Sent 30 1 0 31
115202024-02-20T22:46:14.519Z Done 0 29 30 59
115212024-02-20T22:46:14.519Z Skipped 0 0 0 0
115222024-02-20T22:46:14.519Z Error 0 0 0 0
115232024-02-20T22:46:14.519ZLast Flush: 0 1019 1019
115242024-02-20T22:46:14.519ZDownstairs last five completed:
115252024-02-20T22:46:14.519ZUpstairs last five completed: 30 29 28 27 26
115262024-02-20T22:46:14.519Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:14.034178689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1562}
115272024-02-20T22:46:14.519ZWrote out file "/tmp/test_repair_verify.data"
115282024-02-20T22:46:14.523Z
115292024-02-20T22:46:14.523ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115302024-02-20T22:46:14.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115312024-02-20T22:46:14.544Z 0 000-019 102 102 102 178 178 178 F T T <---
115322024-02-20T22:46:14.544Z 1 020-039 100 100 100 176 176 176 F T T <---
115332024-02-20T22:46:14.544Z 2 040-059 100 106 106 175 185 185 F F F <---
115342024-02-20T22:46:14.544Z 3 060-079 105 106 106 181 185 185 F F F <---
115352024-02-20T22:46:14.544Z 4 080-099 104 106 106 179 183 183 F F F <---
115362024-02-20T22:46:14.544Z 5 100-119 100 100 100 176 176 176 F F F
115372024-02-20T22:46:14.544Z 6 120-139 100 106 106 176 185 185 F F F <---
115382024-02-20T22:46:14.544Z 7 140-159 104 104 104 180 180 180 F F F
115392024-02-20T22:46:14.545Z 8 160-179 105 106 106 181 184 184 F T T <---
115402024-02-20T22:46:14.551Z 9 180-199 102 102 102 178 178 178 F T T <---
115412024-02-20T22:46:14.551Z 10 200-219 97 106 106 170 182 182 F F F <---
115422024-02-20T22:46:14.551Z 11 220-239 105 105 105 181 181 181 F F F
115432024-02-20T22:46:14.551Z 12 240-259 104 104 104 179 179 179 F F F
115442024-02-20T22:46:14.551Z 13 260-279 104 104 104 179 179 179 F T T <---
115452024-02-20T22:46:14.551Z 14 280-299 105 105 105 181 181 181 F T T <---
115462024-02-20T22:46:14.551Z 15 300-319 102 102 102 178 178 178 F F F
115472024-02-20T22:46:14.551Z 16 320-339 105 105 105 181 181 181 F F F
115482024-02-20T22:46:14.552Z 17 340-359 105 105 105 181 181 181 F F F
115492024-02-20T22:46:14.552Z 18 360-379 105 106 106 181 184 184 F F F <---
115502024-02-20T22:46:14.552Z 19 380-399 105 106 106 181 184 184 F F F <---
115512024-02-20T22:46:14.552Z 20 400-419 102 102 102 178 178 178 F F F
115522024-02-20T22:46:14.552Z 21 420-439 101 101 101 177 177 177 F F F
115532024-02-20T22:46:14.552Z 22 440-459 98 106 106 173 184 184 F F F <---
115542024-02-20T22:46:14.552Z 23 460-479 102 102 102 178 178 178 F F F
115552024-02-20T22:46:14.552Z 24 480-499 99 99 99 174 174 174 F F F
115562024-02-20T22:46:14.552Z 25 500-519 100 100 100 176 176 176 F F F
115572024-02-20T22:46:14.552Z 26 520-539 102 106 106 178 182 182 F F F <---
115582024-02-20T22:46:14.552Z 27 540-559 102 102 102 178 178 178 F F F
115592024-02-20T22:46:14.552Z 28 560-579 105 105 105 181 181 181 F F F
115602024-02-20T22:46:14.552Z 29 580-599 105 105 105 181 181 181 F F F
115612024-02-20T22:46:14.552ZMax gen: 106, Max flush: 185
115622024-02-20T22:46:14.552ZError: Difference in extent metadata found!
115632024-02-20T22:46:14.552ZA Difference in extent metadata is expected here
115642024-02-20T22:46:14.552ZOn loop 53
115652024-02-20T22:46:14.552Z
115662024-02-20T22:46:14.552ZVerifying data now
115672024-02-20T22:46:19.253ZNo change after verify
115682024-02-20T22:46:19.253ZLoop: 53 Downstairs dump after verify (and repair):
115692024-02-20T22:46:19.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115702024-02-20T22:46:19.267Z 0 000-019 107 107 107 186 186 186 F F F
115712024-02-20T22:46:19.267Z 1 020-039 107 107 107 186 186 186 F F F
115722024-02-20T22:46:19.267Z 2 040-059 106 106 106 185 185 185 F F F
115732024-02-20T22:46:19.267Z 3 060-079 106 106 106 185 185 185 F F F
115742024-02-20T22:46:19.267Z 4 080-099 106 106 106 183 183 183 F F F
115752024-02-20T22:46:19.267Z 5 100-119 100 100 100 176 176 176 F F F
115762024-02-20T22:46:19.267Z 6 120-139 106 106 106 185 185 185 F F F
115772024-02-20T22:46:19.267Z 7 140-159 104 104 104 180 180 180 F F F
115782024-02-20T22:46:19.267Z 8 160-179 107 107 107 186 186 186 F F F
115792024-02-20T22:46:19.267Z 9 180-199 107 107 107 186 186 186 F F F
115802024-02-20T22:46:19.267Z 10 200-219 106 106 106 182 182 182 F F F
115812024-02-20T22:46:19.267Z 11 220-239 105 105 105 181 181 181 F F F
115822024-02-20T22:46:19.267Z 12 240-259 104 104 104 179 179 179 F F F
115832024-02-20T22:46:19.277Z 13 260-279 107 107 107 186 186 186 F F F
115842024-02-20T22:46:19.277Z 14 280-299 107 107 107 186 186 186 F F F
115852024-02-20T22:46:19.278Z 15 300-319 102 102 102 178 178 178 F F F
115862024-02-20T22:46:19.278Z 16 320-339 105 105 105 181 181 181 F F F
115872024-02-20T22:46:19.278Z 17 340-359 105 105 105 181 181 181 F F F
115882024-02-20T22:46:19.278Z 18 360-379 106 106 106 184 184 184 F F F
115892024-02-20T22:46:19.278Z 19 380-399 106 106 106 184 184 184 F F F
115902024-02-20T22:46:19.278Z 20 400-419 102 102 102 178 178 178 F F F
115912024-02-20T22:46:19.278Z 21 420-439 101 101 101 177 177 177 F F F
115922024-02-20T22:46:19.278Z 22 440-459 106 106 106 184 184 184 F F F
115932024-02-20T22:46:19.278Z 23 460-479 102 102 102 178 178 178 F F F
115942024-02-20T22:46:19.278Z 24 480-499 99 99 99 174 174 174 F F F
115952024-02-20T22:46:19.278Z 25 500-519 100 100 100 176 176 176 F F F
115962024-02-20T22:46:19.278Z 26 520-539 106 106 106 182 182 182 F F F
115972024-02-20T22:46:19.278Z 27 540-559 102 102 102 178 178 178 F F F
115982024-02-20T22:46:19.278Z 28 560-579 105 105 105 181 181 181 F F F
115992024-02-20T22:46:19.278Z 29 580-599 105 105 105 181 181 181 F F F
116002024-02-20T22:46:19.278ZMax gen: 107, Max flush: 186
116012024-02-20T22:46:19.278Z
116022024-02-20T22:46:19.278ZBegin loop 54
116032024-02-20T22:46:19.278ZDownstairs to restart: 0
116042024-02-20T22:46:19.289Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.807752427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570}
116052024-02-20T22:46:19.289Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808014297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570}
116062024-02-20T22:46:19.289Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808040126Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570}
116072024-02-20T22:46:19.289Z{"msg":"Crucible b7017688-5f6b-42f1-9624-bec7aeb5f86c has session id: 0539d3ac-c638-4cd3-8e7d-a915ec795377","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808054666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116082024-02-20T22:46:19.290Z{"msg":"Upstairs opts: Upstairs UUID: b7017688-5f6b-42f1-9624-bec7aeb5f86c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808111874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116092024-02-20T22:46:19.290Z{"msg":"Crucible stats registered with UUID: b7017688-5f6b-42f1-9624-bec7aeb5f86c","v":0,"name":"crucible","level":30Crucible runtime is spawned
116102024-02-20T22:46:19.290Z,"time":"2024-02-20T22:46:18.808149112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116112024-02-20T22:46:19.290Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808232669Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570}
116122024-02-20T22:46:19.290Z{"msg":"b7017688-5f6b-42f1-9624-bec7aeb5f86c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808352865Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116132024-02-20T22:46:19.290Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808377984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116142024-02-20T22:46:19.290Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808398993Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116152024-02-20T22:46:19.290Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808434172Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116162024-02-20T22:46:19.290Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808464591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116172024-02-20T22:46:19.290Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.80850353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116182024-02-20T22:46:19.290Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808531159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116192024-02-20T22:46:19.290Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808596066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116202024-02-20T22:46:19.290Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808646084Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116212024-02-20T22:46:19.290Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808677743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116222024-02-20T22:46:19.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808704422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116232024-02-20T22:46:19.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.808731012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116242024-02-20T22:46:19.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.80877549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116252024-02-20T22:46:19.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809384368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116262024-02-20T22:46:19.290Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809423857Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116272024-02-20T22:46:19.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809443216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116282024-02-20T22:46:19.312Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809469785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116292024-02-20T22:46:19.312Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809540703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116302024-02-20T22:46:19.312Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809572042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116312024-02-20T22:46:19.312Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.809997456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116322024-02-20T22:46:19.312Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810058574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116332024-02-20T22:46:19.312Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810330254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116342024-02-20T22:46:19.312Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810620834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116352024-02-20T22:46:19.312Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810686091Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116362024-02-20T22:46:19.312Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810743849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116372024-02-20T22:46:19.313Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810771388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116382024-02-20T22:46:19.313Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810800258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116392024-02-20T22:46:19.313Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810827026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116402024-02-20T22:46:19.313Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810851896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116412024-02-20T22:46:19.313Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810873525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116422024-02-20T22:46:19.313Z{"msg":"[0]R flush_numbers[0..12]: [186, 186, 185, 185, 183, 176, 185, 180, 186, 186, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810898614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116432024-02-20T22:46:19.313Z{"msg":"[0]R generation[0..12]: [107, 107, 106, 106, 106, 100, 106, 104, 107, 107, 106, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810913574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116442024-02-20T22:46:19.313Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810929813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116452024-02-20T22:46:19.313Z{"msg":"[1]R flush_numbers[0..12]: [186, 186, 185, 185, 183, 176, 185, 180, 186, 186, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810952202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116462024-02-20T22:46:19.313Z{"msg":"[1]R generation[0..12]: [107, 107, 106, 106, 106, 100, 106, 104, 107, 107, 106, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810966581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116472024-02-20T22:46:19.313Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.810987661Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116482024-02-20T22:46:19.313Z{"msg":"[2]R flush_numbers[0..12]: [186, 186, 185, 185, 183, 176, 185, 180, 186, 186, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.81102541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116492024-02-20T22:46:19.313Z{"msg":"[2]R generation[0..12]: [107, 107, 106, 106, 106, 100, 106, 104, 107, 107, 106, 105]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
116502024-02-20T22:46:19.313Z,"level":30,"time":"2024-02-20T22:46:18.811045219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116512024-02-20T22:46:19.313Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811067348Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116522024-02-20T22:46:19.313Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811081168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116532024-02-20T22:46:19.313Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811101667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116542024-02-20T22:46:19.313Z{"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811115146Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116552024-02-20T22:46:19.313Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811134835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116562024-02-20T22:46:19.313Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811148575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116572024-02-20T22:46:19.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811167654Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116582024-02-20T22:46:19.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811181374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116592024-02-20T22:46:19.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811195783Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116602024-02-20T22:46:19.314Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811216433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116612024-02-20T22:46:19.314Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811229292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116622024-02-20T22:46:19.314Z{"msg":"b7017688-5f6b-42f1-9624-bec7aeb5f86c is now active with session: 0539d3ac-c638-4cd3-8e7d-a915ec795377","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811242672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116632024-02-20T22:46:19.314Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811254911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570,"session_id":"0539d3ac-c638-4cd3-8e7d-a915ec795377"}
116642024-02-20T22:46:19.314Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:18.811267141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570}
116652024-02-20T22:46:19.314ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116662024-02-20T22:46:19.314ZLoading write count information from file "/tmp/test_repair_verify.data"
116672024-02-20T22:46:19.314ZRun Repair workload
116682024-02-20T22:46:19.314Z01/30 Write block 102 len 28672 data: 4 3 5 5 5 5 7
116692024-02-20T22:46:19.314Z02/30 Read block 24 len 32768
116702024-02-20T22:46:19.314Z03/30 Write block 523 len 20480 data: 6 4 5 8 9
116712024-02-20T22:46:19.314Z04/30 Flush
116722024-02-20T22:46:19.314Z05/30 Write block 389 len 20480 data: 8 9 8 10 11
116732024-02-20T22:46:19.314Z06/30 Read block 416 len 32768
116742024-02-20T22:46:19.314Z07/30 Write block 491 len 24576 data: 6 9 11 11 11 11
116752024-02-20T22:46:19.314Z08/30 Read block 559 len 12288
116762024-02-20T22:46:19.314Z09/30 Write block 4 len 8192 data: 9 9
116772024-02-20T22:46:19.314Z10/30 Write block 573 len 8192 data: 6 6
116782024-02-20T22:46:19.314Z11/30 Write block 230 len 8192 data: 15 13
116792024-02-20T22:46:19.314Z12/30 Read block 280 len 36864
116802024-02-20T22:46:19.314Z13/30 Flush
116812024-02-20T22:46:19.314Z14/30 Write block 325 len 32768 data: 9 10 9 9 9 8 8 12
116822024-02-20T22:46:19.314Z15/30 Write block 491 len 12288 data: 7 10 12
116832024-02-20T22:46:19.314Z16/30 Read block 200 len 16384
116842024-02-20T22:46:19.314Z17/30 Read block 502 len 36864
116852024-02-20T22:46:19.324Z18/30 Flush
116862024-02-20T22:46:19.324Z19/30 Write block 325 len 16384 data: 10 11 10 10
116872024-02-20T22:46:19.324Z20/30 Write block 60 len 24576 data: 5 4 5 6 6 6
116882024-02-20T22:46:19.324Z21/30 Write block 432 len 8192 data: 3 3
116892024-02-20T22:46:19.324Z22/30 Write block 310 len 32768 data: 10 10 9 8 10 12 11 13
116902024-02-20T22:46:19.324Z23/30 Flush
116912024-02-20T22:46:19.324Z24/30 Write block 332 len 28672 data: 13 14 12 10 11 12 13
116922024-02-20T22:46:19.324Z25/30 Write block 459 len 28672 data: 14 12 14 14 14 11 11
116932024-02-20T22:46:19.324Z26/30 Read block 41 len 32768
116942024-02-20T22:46:19.324Z27/30 Write block 239 len 28672 data: 6 4 3 2 2 2 4
116952024-02-20T22:46:19.324Z28/30 Write block 181 len 40960 data: 13 12 9 10 8 8 9 8 7 6
116962024-02-20T22:46:19.331Z29/30 Write block 413 len 32768 data: 13 13 12 10 8 9 12 12
116972024-02-20T22:46:19.331Z30/30 Read block 253 len 4096
116982024-02-20T22:46:19.331Z----------------------------------------------------------------
116992024-02-20T22:46:19.331Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
117002024-02-20T22:46:19.331ZStates: Active Active Active
117012024-02-20T22:46:19.331ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117022024-02-20T22:46:19.331Z 1 Acked 1000 Write 7 Done Done Done false
117032024-02-20T22:46:19.331Z 2 Acked 1001 Read 8 Done Done Done false
117042024-02-20T22:46:19.331Z 3 Acked 1002 Write 5 Sent Done Done false
117052024-02-20T22:46:19.332Z 4 Acked 1003 Flush 0 Sent Done Done false
117062024-02-20T22:46:19.332Z 5 Acked 1004 Write 5 Sent Done Done false
117072024-02-20T22:46:19.332Z 6 Acked 1005 Read 8 Sent Done Done false
117082024-02-20T22:46:19.332Z 7 Acked 1006 Write 6 Sent Done Done false
117092024-02-20T22:46:19.332Z 8 Acked 1007 Read 3 Sent Done Done false
117102024-02-20T22:46:19.332Z 9 Acked 1008 Write 2 Sent Done Done false
117112024-02-20T22:46:19.332Z 10 Acked 1009 Write 2 Sent Done Done false
117122024-02-20T22:46:19.332Z 11 Acked 1010 Write 2 Sent Done Done false
117132024-02-20T22:46:19.332Z 12 Acked 1011 Read 9 Sent Done Done false
117142024-02-20T22:46:19.332Z 13 Acked 1012 Flush 0 Sent Done Done false
117152024-02-20T22:46:19.332Z 14 Acked 1013 Write 8 Sent Done Done false
117162024-02-20T22:46:19.332Z 15 Acked 1014 Write 3 Sent Done Done false
117172024-02-20T22:46:19.332Z 16 Acked 1015 Read 4 Sent Done Done false
117182024-02-20T22:46:19.332Z 17 Acked 1016 Read 9 Sent Done Done false
117192024-02-20T22:46:19.332Z 18 Acked 1017 Flush 0 Sent Done Done false
117202024-02-20T22:46:19.332Z 19 Acked 1018 Write 4 Sent Done Done false
117212024-02-20T22:46:19.332Z 20 Acked 1019 Write 6 Sent Done Done false
117222024-02-20T22:46:19.332Z 21 Acked 1020 Write 2 Sent Done Done false
117232024-02-20T22:46:19.332Z 22 Acked 1021 Write 8 Sent Done Done false
117242024-02-20T22:46:19.332Z 23 Acked 1022 Flush 0 Sent Done Done false
117252024-02-20T22:46:19.332Z 24 Acked 1023 Write 7 Sent Done Done false
117262024-02-20T22:46:19.332Z 25 Acked 1024 Write 7 Sent Done Done false
117272024-02-20T22:46:19.332Z 26 Acked 1025 Read 8 Sent Done Done false
117282024-02-20T22:46:19.332Z 27 Acked 1026 Write 7 Sent Done Done false
117292024-02-20T22:46:19.333Z 28 Acked 1027 Write 10 Sent Done Done false
117302024-02-20T22:46:19.333Z 29 Acked 1028 Write 8 Sent Done Done false
117312024-02-20T22:46:19.333Z 30 Acked 1029 Read 1 Sent Done Done false
117322024-02-20T22:46:19.333Z STATES DS:0 DS:1 DS:2 TOTAL
117332024-02-20T22:46:19.333Z New 0 0 0 0
117342024-02-20T22:46:19.333Z Sent 28 0 0 28
117352024-02-20T22:46:19.333Z Done 2 30 30 62
117362024-02-20T22:46:19.333Z Skipped 0 0 0 0
117372024-02-20T22:46:19.333Z Error 0 0 0 0
117382024-02-20T22:46:19.333ZLast Flush: 0 1022 1022
117392024-02-20T22:46:19.333ZDownstairs last five completed:
117402024-02-20T22:46:19.333ZUpstairs last five completed: 30 29 28 27 26
117412024-02-20T22:46:19.333Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:18.847175806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1570}
117422024-02-20T22:46:19.333ZWrote out file "/tmp/test_repair_verify.data"
117432024-02-20T22:46:19.333Z
117442024-02-20T22:46:19.339ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117452024-02-20T22:46:19.351ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117462024-02-20T22:46:19.351Z 0 000-019 107 108 108 186 188 188 F F F <---
117472024-02-20T22:46:19.351Z 1 020-039 107 107 107 186 186 186 F F F
117482024-02-20T22:46:19.351Z 2 040-059 106 106 106 185 185 185 F F F
117492024-02-20T22:46:19.351Z 3 060-079 106 108 108 185 190 190 F F F <---
117502024-02-20T22:46:19.351Z 4 080-099 106 106 106 183 183 183 F F F
117512024-02-20T22:46:19.351Z 5 100-119 100 108 108 176 187 187 T F F <---
117522024-02-20T22:46:19.351Z 6 120-139 106 106 106 185 185 185 F F F
117532024-02-20T22:46:19.351Z 7 140-159 104 104 104 180 180 180 F F F
117542024-02-20T22:46:19.351Z 8 160-179 107 107 107 186 186 186 F F F
117552024-02-20T22:46:19.351Z 9 180-199 107 107 107 186 186 186 F T T <---
117562024-02-20T22:46:19.358Z 10 200-219 106 106 106 182 182 182 F F F
117572024-02-20T22:46:19.358Z 11 220-239 105 108 108 181 188 188 F T T <---
117582024-02-20T22:46:19.358Z 12 240-259 104 104 104 179 179 179 F T T <---
117592024-02-20T22:46:19.358Z 13 260-279 107 107 107 186 186 186 F F F
117602024-02-20T22:46:19.358Z 14 280-299 107 107 107 186 186 186 F F F
117612024-02-20T22:46:19.359Z 15 300-319 102 108 108 178 190 190 F F F <---
117622024-02-20T22:46:19.359Z 16 320-339 105 108 108 181 190 190 F T T <---
117632024-02-20T22:46:19.359Z 17 340-359 105 105 105 181 181 181 F F F
117642024-02-20T22:46:19.359Z 18 360-379 106 106 106 184 184 184 F F F
117652024-02-20T22:46:19.359Z 19 380-399 106 108 108 184 188 188 F F F <---
117662024-02-20T22:46:19.359Z 20 400-419 102 102 102 178 178 178 F T T <---
117672024-02-20T22:46:19.359Z 21 420-439 101 108 108 177 190 190 F T T <---
117682024-02-20T22:46:19.359Z 22 440-459 106 106 106 184 184 184 F T T <---
117692024-02-20T22:46:19.359Z 23 460-479 102 102 102 178 178 178 F T T <---
117702024-02-20T22:46:19.359Z 24 480-499 99 108 108 174 189 189 F F F <---
117712024-02-20T22:46:19.359Z 25 500-519 100 100 100 176 176 176 F F F
117722024-02-20T22:46:19.359Z 26 520-539 106 108 108 182 187 187 F F F <---
117732024-02-20T22:46:19.359Z 27 540-559 102 102 102 178 178 178 F F F
117742024-02-20T22:46:19.359Z 28 560-579 105 108 108 181 188 188 F F F <---
117752024-02-20T22:46:19.359Z 29 580-599 105 105 105 181 181 181 F F F
117762024-02-20T22:46:19.359ZMax gen: 108, Max flush: 190
117772024-02-20T22:46:19.359ZError: Difference in extent metadata found!
117782024-02-20T22:46:19.359ZA Difference in extent metadata is expected here
117792024-02-20T22:46:19.359ZOn loop 54
117802024-02-20T22:46:19.359Z
117812024-02-20T22:46:19.359ZVerifying data now
117822024-02-20T22:46:24.147ZNo change after verify
117832024-02-20T22:46:24.147ZLoop: 54 Downstairs dump after verify (and repair):
117842024-02-20T22:46:24.171ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117852024-02-20T22:46:24.171Z 0 000-019 108 108 108 188 188 188 F F F
117862024-02-20T22:46:24.171Z 1 020-039 107 107 107 186 186 186 F F F
117872024-02-20T22:46:24.171Z 2 040-059 106 106 106 185 185 185 F F F
117882024-02-20T22:46:24.171Z 3 060-079 108 108 108 190 190 190 F F F
117892024-02-20T22:46:24.171Z 4 080-099 106 106 106 183 183 183 F F F
117902024-02-20T22:46:24.178Z 5 100-119 108 108 108 187 187 187 F F F
117912024-02-20T22:46:24.178Z 6 120-139 106 106 106 185 185 185 F F F
117922024-02-20T22:46:24.178Z 7 140-159 104 104 104 180 180 180 F F F
117932024-02-20T22:46:24.178Z 8 160-179 107 107 107 186 186 186 F F F
117942024-02-20T22:46:24.178Z 9 180-199 109 109 109 191 191 191 F F F
117952024-02-20T22:46:24.178Z 10 200-219 106 106 106 182 182 182 F F F
117962024-02-20T22:46:24.178Z 11 220-239 109 109 109 191 191 191 F F F
117972024-02-20T22:46:24.178Z 12 240-259 109 109 109 191 191 191 F F F
117982024-02-20T22:46:24.178Z 13 260-279 107 107 107 186 186 186 F F F
117992024-02-20T22:46:24.178Z 14 280-299 107 107 107 186 186 186 F F F
118002024-02-20T22:46:24.178Z 15 300-319 108 108 108 190 190 190 F F F
118012024-02-20T22:46:24.178Z 16 320-339 109 109 109 191 191 191 F F F
118022024-02-20T22:46:24.178Z 17 340-359 105 105 105 181 181 181 F F F
118032024-02-20T22:46:24.178Z 18 360-379 106 106 106 184 184 184 F F F
118042024-02-20T22:46:24.178Z 19 380-399 108 108 108 188 188 188 F F F
118052024-02-20T22:46:24.178Z 20 400-419 109 109 109 191 191 191 F F F
118062024-02-20T22:46:24.178Z 21 420-439 109 109 109 191 191 191 F F F
118072024-02-20T22:46:24.178Z 22 440-459 109 109 109 191 191 191 F F F
118082024-02-20T22:46:24.178Z 23 460-479 109 109 109 191 191 191 F F F
118092024-02-20T22:46:24.178Z 24 480-499 108 108 108 189 189 189 F F F
118102024-02-20T22:46:24.178Z 25 500-519 100 100 100 176 176 176 F F F
118112024-02-20T22:46:24.178Z 26 520-539 108 108 108 187 187 187 F F F
118122024-02-20T22:46:24.179Z 27 540-559 102 102 102 178 178 178 F F F
118132024-02-20T22:46:24.179Z 28 560-579 108 108 108 188 188 188 F F F
118142024-02-20T22:46:24.179Z 29 580-599 105 105 105 181 181 181 F F F
118152024-02-20T22:46:24.179ZMax gen: 109, Max flush: 191
118162024-02-20T22:46:24.179Z
118172024-02-20T22:46:24.179ZBegin loop 55
118182024-02-20T22:46:24.179ZDownstairs to restart: 0
118192024-02-20T22:46:24.192Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699297852Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578}
118202024-02-20T22:46:24.192Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699569502Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578}
118212024-02-20T22:46:24.192Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699596982Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578}
118222024-02-20T22:46:24.192ZCrucible runtime is spawned
118232024-02-20T22:46:24.192Z{"msg":"Crucible b0fe1aa5-cbd9-49c1-bd8d-d565a0916f73 has session id: 49aebb25-276d-40a2-99b0-139a949a6d06","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.69964548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118242024-02-20T22:46:24.192Z{"msg":"Upstairs opts: Upstairs UUID: b0fe1aa5-cbd9-49c1-bd8d-d565a0916f73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699697238Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118252024-02-20T22:46:24.193Z{"msg":"Crucible stats registered with UUID: b0fe1aa5-cbd9-49c1-bd8d-d565a0916f73","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699728337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118262024-02-20T22:46:24.193Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699745306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578}
118272024-02-20T22:46:24.193Z{"msg":"b0fe1aa5-cbd9-49c1-bd8d-d565a0916f73 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699769175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118282024-02-20T22:46:24.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699791804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118292024-02-20T22:46:24.193Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699829773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118302024-02-20T22:46:24.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699869272Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118312024-02-20T22:46:24.193Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699894971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118322024-02-20T22:46:24.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.699970758Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118332024-02-20T22:46:24.193Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700020066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118342024-02-20T22:46:24.193Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700060755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118352024-02-20T22:46:24.193Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700104203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118362024-02-20T22:46:24.193Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700134072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118372024-02-20T22:46:24.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70018576Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118382024-02-20T22:46:24.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700231119Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118392024-02-20T22:46:24.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700305956Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118402024-02-20T22:46:24.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.700963153Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118412024-02-20T22:46:24.193Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.701009471Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118422024-02-20T22:46:24.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70103721Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118432024-02-20T22:46:24.193Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.701066189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118442024-02-20T22:46:24.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.701090748Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118452024-02-20T22:46:24.193Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.701124317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118462024-02-20T22:46:24.193Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.701540292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118472024-02-20T22:46:24.193Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70158481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118482024-02-20T22:46:24.194Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.701833181Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118492024-02-20T22:46:24.194Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702035644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118502024-02-20T22:46:24.194Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702071613Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118512024-02-20T22:46:24.194Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702088722Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118522024-02-20T22:46:24.194Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702120781Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118532024-02-20T22:46:24.194Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70215248Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118542024-02-20T22:46:24.194Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702203438Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118552024-02-20T22:46:24.194Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702358423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118562024-02-20T22:46:24.194Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702405171Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118572024-02-20T22:46:24.194Z{"msg":"[0]R flush_numbers[0..12]: [188, 186, 185, 190, 183, 187, 185, 180, 186, 191, 182, 191]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70242563Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118582024-02-20T22:46:24.194Z{"msg":"[0]R generation[0..12]: [108, 107, 106, 108, 106, 108, 106, 104, 107, 109, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702448919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118592024-02-20T22:46:24.194Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702464639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118602024-02-20T22:46:24.194Z{"msg":"[1]R flush_numbers[0..12]: [188, 186, 185, 190, 183, 187, 185, 180, 186, 191, 182, 191]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702486168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118612024-02-20T22:46:24.194Z{"msg":"[1]R generation[0..12]: [108, 107, 106, 108, 106, 108, 106, 104, 107, 109, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702502278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118622024-02-20T22:46:24.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702516877Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118632024-02-20T22:46:24.194Z{"msg":"[2]R flush_numbers[0..12]: [188, 186, 185, 190, 183, 187, 185, 180, 186, 191, 182, 191]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702538496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118642024-02-20T22:46:24.194Z{"msg":"[2]R generation[0..12]: [108, 107, 106, 108, 106, 108, 106, 104, 107, 109, 106, 109]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
118652024-02-20T22:46:24.194Z,"time":"2024-02-20T22:46:23.702552856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118662024-02-20T22:46:24.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702569635Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118672024-02-20T22:46:24.194Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702597044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118682024-02-20T22:46:24.194Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702611263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118692024-02-20T22:46:24.194Z{"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702625033Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118702024-02-20T22:46:24.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702638433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118712024-02-20T22:46:24.194Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702651952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118722024-02-20T22:46:24.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702664032Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118732024-02-20T22:46:24.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702686171Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118742024-02-20T22:46:24.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70270029Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118752024-02-20T22:46:24.195Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.70272171Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118762024-02-20T22:46:24.195Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702734609Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118772024-02-20T22:46:24.195Z{"msg":"b0fe1aa5-cbd9-49c1-bd8d-d565a0916f73 is now active with session: 49aebb25-276d-40a2-99b0-139a949a6d06","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702755678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118782024-02-20T22:46:24.195Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702768128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578,"session_id":"49aebb25-276d-40a2-99b0-139a949a6d06"}
118792024-02-20T22:46:24.195Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:23.702788507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578}
118802024-02-20T22:46:24.195ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118812024-02-20T22:46:24.195ZLoading write count information from file "/tmp/test_repair_verify.data"
118822024-02-20T22:46:24.195ZRun Repair workload
118832024-02-20T22:46:24.195Z01/30 Write block 35 len 12288 data: 12 11 10
118842024-02-20T22:46:24.195Z02/30 Write block 298 len 20480 data: 9 9 8 12 14
118852024-02-20T22:46:24.195Z03/30 Read block 323 len 12288
118862024-02-20T22:46:24.199Z04/30 Read block 386 len 20480
118872024-02-20T22:46:24.200Z05/30 Read block 444 len 36864
118882024-02-20T22:46:24.200Z06/30 Write block 504 len 16384 data: 7 9 8 8
118892024-02-20T22:46:24.200Z07/30 Write block 356 len 20480 data: 6 4 6 9 11
118902024-02-20T22:46:24.200Z08/30 Read block 511 len 12288
118912024-02-20T22:46:24.200Z09/30 Write block 362 len 40960 data: 12 15 14 16 14 10 9 10 13 15
118922024-02-20T22:46:24.200Z10/30 Write block 392 len 4096 data: 11
118932024-02-20T22:46:24.200Z11/30 Write block 343 len 36864 data: 10 6 6 6 8 9 7 7 6
118942024-02-20T22:46:24.200Z12/30 Read block 156 len 12288
118952024-02-20T22:46:24.200Z13/30 Read block 43 len 12288
118962024-02-20T22:46:24.200Z14/30 Read block 470 len 28672
118972024-02-20T22:46:24.200Z15/30 Write block 239 len 28672 data: 7 5 4 3 3 3 5
118982024-02-20T22:46:24.200Z16/30 Flush
118992024-02-20T22:46:24.205Z17/30 Write block 94 len 8192 data: 7 6
119002024-02-20T22:46:24.205Z18/30 Read block 369 len 20480
119012024-02-20T22:46:24.205Z19/30 Write block 18 len 20480 data: 11 11 11 8 9
119022024-02-20T22:46:24.210Z20/30 Write block 413 len 32768 data: 14 14 13 11 9 10 13 13
119032024-02-20T22:46:24.211Z21/30 Read block 353 len 36864
119042024-02-20T22:46:24.211Z22/30 Read block 549 len 12288
119052024-02-20T22:46:24.211Z23/30 Flush
119062024-02-20T22:46:24.215Z24/30 Write block 121 len 16384 data: 10 11 9 9
119072024-02-20T22:46:24.215Z25/30 Write block 310 len 36864 data: 11 11 10 9 11 13 12 14 10
119082024-02-20T22:46:24.215Z26/30 Write block 485 len 28672 data: 10 8 7 8 6 6 8
119092024-02-20T22:46:24.215Z27/30 Read block 348 len 28672
119102024-02-20T22:46:24.215Z28/30 Write block 147 len 32768 data: 6 8 10 13 14 12 7 7
119112024-02-20T22:46:24.215Z29/30 Read block 368 len 8192
119122024-02-20T22:46:24.219Z30/30 Read block 457 len 12288
119132024-02-20T22:46:24.219Z----------------------------------------------------------------
119142024-02-20T22:46:24.219Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
119152024-02-20T22:46:24.219ZStates: Active Active Active
119162024-02-20T22:46:24.219ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119172024-02-20T22:46:24.220Z 1 Acked 1000 Write 3 Done Done Done false
119182024-02-20T22:46:24.220Z 2 Acked 1001 Write 5 Sent Done Done false
119192024-02-20T22:46:24.220Z 3 Acked 1002 Read 3 Sent Done Done false
119202024-02-20T22:46:24.220Z 4 Acked 1003 Read 5 Sent Done Done false
119212024-02-20T22:46:24.220Z 5 Acked 1004 Read 9 Sent Done Done false
119222024-02-20T22:46:24.220Z 6 Acked 1005 Write 4 Sent Done Done false
119232024-02-20T22:46:24.220Z 7 Acked 1006 Write 5 Sent Done Done false
119242024-02-20T22:46:24.220Z 8 Acked 1007 Read 3 Sent Done Done false
119252024-02-20T22:46:24.220Z 9 Acked 1008 Write 10 Sent Done Done false
119262024-02-20T22:46:24.220Z 10 Acked 1009 Write 1 Sent Done Done false
119272024-02-20T22:46:24.220Z 11 Acked 1010 Write 9 Sent Done Done false
119282024-02-20T22:46:24.220Z 12 Acked 1011 Read 3 Sent Done Done false
119292024-02-20T22:46:24.220Z 13 Acked 1012 Read 3 Sent Done Done false
119302024-02-20T22:46:24.220Z 14 Acked 1013 Read 7 Sent Done Done false
119312024-02-20T22:46:24.220Z 15 Acked 1014 Write 7 Sent Done Done false
119322024-02-20T22:46:24.220Z 16 Acked 1015 Flush 0 Sent Done Done false
119332024-02-20T22:46:24.220Z 17 Acked 1016 Write 2 Sent Done Done false
119342024-02-20T22:46:24.220Z 18 Acked 1017 Read 5 Sent Done Done false
119352024-02-20T22:46:24.220Z 19 Acked 1018 Write 5 Sent Done Done false
119362024-02-20T22:46:24.220Z 20 Acked 1019 Write 8 Sent Done Done false
119372024-02-20T22:46:24.220Z 21 Acked 1020 Read 9 Sent Done Done false
119382024-02-20T22:46:24.220Z 22 Acked 1021 Read 3 Sent Done Done false
119392024-02-20T22:46:24.220Z 23 Acked 1022 Flush 0 Sent Done Done false
119402024-02-20T22:46:24.220Z 24 Acked 1023 Write 4 Sent Done Done false
119412024-02-20T22:46:24.220Z 25 Acked 1024 Write 9 Sent Done Done false
119422024-02-20T22:46:24.221Z 26 Acked 1025 Write 7 Sent Done Done false
119432024-02-20T22:46:24.221Z 27 Acked 1026 Read 7 Sent Done Done false
119442024-02-20T22:46:24.221Z 28 Acked 1027 Write 8 Sent Done Done false
119452024-02-20T22:46:24.221Z 29 Acked 1028 Read 2 Sent Done Done false
119462024-02-20T22:46:24.221Z 30 Acked 1029 Read 3 Sent Sent Done false
119472024-02-20T22:46:24.221Z STATES DS:0 DS:1 DS:2 TOTAL
119482024-02-20T22:46:24.221Z New 0 0 0 0
119492024-02-20T22:46:24.221Z Sent 29 1 0 30
119502024-02-20T22:46:24.221Z Done 1 29 30 60
119512024-02-20T22:46:24.221Z Skipped 0 0 0 0
119522024-02-20T22:46:24.221Z Error 0 0 0 0
119532024-02-20T22:46:24.221ZLast Flush: 0 1022 1022
119542024-02-20T22:46:24.221ZDownstairs last five completed:
119552024-02-20T22:46:24.221ZUpstairs last five completed: 30 29 28 27 26
119562024-02-20T22:46:24.221Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:23.735629492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1578}
119572024-02-20T22:46:24.221ZWrote out file "/tmp/test_repair_verify.data"
119582024-02-20T22:46:24.225Z
119592024-02-20T22:46:24.225ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119602024-02-20T22:46:24.238ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119612024-02-20T22:46:24.238Z 0 000-019 108 110 110 188 193 193 F F F <---
119622024-02-20T22:46:24.238Z 1 020-039 107 110 110 186 193 193 T F F <---
119632024-02-20T22:46:24.238Z 2 040-059 106 106 106 185 185 185 F F F
119642024-02-20T22:46:24.238Z 3 060-079 108 108 108 190 190 190 F F F
119652024-02-20T22:46:24.238Z 4 080-099 106 110 110 183 193 193 F F F <---
119662024-02-20T22:46:24.238Z 5 100-119 108 108 108 187 187 187 F F F
119672024-02-20T22:46:24.238Z 6 120-139 106 106 106 185 185 185 F T T <---
119682024-02-20T22:46:24.239Z 7 140-159 104 104 104 180 180 180 F T T <---
119692024-02-20T22:46:24.239Z 8 160-179 107 107 107 186 186 186 F F F
119702024-02-20T22:46:24.239Z 9 180-199 109 109 109 191 191 191 F F F
119712024-02-20T22:46:24.239Z 10 200-219 106 106 106 182 182 182 F F F
119722024-02-20T22:46:24.239Z 11 220-239 109 110 110 191 192 192 F F F <---
119732024-02-20T22:46:24.239Z 12 240-259 109 110 110 191 192 192 F F F <---
119742024-02-20T22:46:24.239Z 13 260-279 107 107 107 186 186 186 F F F
119752024-02-20T22:46:24.245Z 14 280-299 107 110 110 186 192 192 F F F <---
119762024-02-20T22:46:24.245Z 15 300-319 108 110 110 190 192 192 F T T <---
119772024-02-20T22:46:24.245Z 16 320-339 109 109 109 191 191 191 F F F
119782024-02-20T22:46:24.245Z 17 340-359 105 110 110 181 192 192 F F F <---
119792024-02-20T22:46:24.245Z 18 360-379 106 110 110 184 192 192 F F F <---
119802024-02-20T22:46:24.245Z 19 380-399 108 110 110 188 192 192 F F F <---
119812024-02-20T22:46:24.245Z 20 400-419 109 110 110 191 193 193 F F F <---
119822024-02-20T22:46:24.245Z 21 420-439 109 110 110 191 193 193 F F F <---
119832024-02-20T22:46:24.245Z 22 440-459 109 109 109 191 191 191 F F F
119842024-02-20T22:46:24.245Z 23 460-479 109 109 109 191 191 191 F F F
119852024-02-20T22:46:24.245Z 24 480-499 108 108 108 189 189 189 F T T <---
119862024-02-20T22:46:24.245Z 25 500-519 100 110 110 176 192 192 F F F <---
119872024-02-20T22:46:24.245Z 26 520-539 108 108 108 187 187 187 F F F
119882024-02-20T22:46:24.245Z 27 540-559 102 102 102 178 178 178 F F F
119892024-02-20T22:46:24.245Z 28 560-579 108 108 108 188 188 188 F F F
119902024-02-20T22:46:24.245Z 29 580-599 105 105 105 181 181 181 F F F
119912024-02-20T22:46:24.245ZMax gen: 110, Max flush: 193
119922024-02-20T22:46:24.245ZError: Difference in extent metadata found!
119932024-02-20T22:46:24.245ZA Difference in extent metadata is expected here
119942024-02-20T22:46:24.246ZOn loop 55
119952024-02-20T22:46:24.246Z
119962024-02-20T22:46:24.246ZVerifying data now
119972024-02-20T22:46:29.006ZNo change after verify
119982024-02-20T22:46:29.006ZLoop: 55 Downstairs dump after verify (and repair):
119992024-02-20T22:46:29.020ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120002024-02-20T22:46:29.020Z 0 000-019 110 110 110 193 193 193 F F F
120012024-02-20T22:46:29.020Z 1 020-039 110 110 110 193 193 193 F F F
120022024-02-20T22:46:29.020Z 2 040-059 106 106 106 185 185 185 F F F
120032024-02-20T22:46:29.020Z 3 060-079 108 108 108 190 190 190 F F F
120042024-02-20T22:46:29.020Z 4 080-099 110 110 110 193 193 193 F F F
120052024-02-20T22:46:29.020Z 5 100-119 108 108 108 187 187 187 F F F
120062024-02-20T22:46:29.020Z 6 120-139 111 111 111 194 194 194 F F F
120072024-02-20T22:46:29.020Z 7 140-159 111 111 111 194 194 194 F F F
120082024-02-20T22:46:29.020Z 8 160-179 107 107 107 186 186 186 F F F
120092024-02-20T22:46:29.025Z 9 180-199 109 109 109 191 191 191 F F F
120102024-02-20T22:46:29.025Z 10 200-219 106 106 106 182 182 182 F F F
120112024-02-20T22:46:29.025Z 11 220-239 110 110 110 192 192 192 F F F
120122024-02-20T22:46:29.025Z 12 240-259 110 110 110 192 192 192 F F F
120132024-02-20T22:46:29.025Z 13 260-279 107 107 107 186 186 186 F F F
120142024-02-20T22:46:29.025Z 14 280-299 110 110 110 192 192 192 F F F
120152024-02-20T22:46:29.025Z 15 300-319 111 111 111 194 194 194 F F F
120162024-02-20T22:46:29.025Z 16 320-339 109 109 109 191 191 191 F F F
120172024-02-20T22:46:29.025Z 17 340-359 110 110 110 192 192 192 F F F
120182024-02-20T22:46:29.025Z 18 360-379 110 110 110 192 192 192 F F F
120192024-02-20T22:46:29.025Z 19 380-399 110 110 110 192 192 192 F F F
120202024-02-20T22:46:29.025Z 20 400-419 110 110 110 193 193 193 F F F
120212024-02-20T22:46:29.025Z 21 420-439 110 110 110 193 193 193 F F F
120222024-02-20T22:46:29.025Z 22 440-459 109 109 109 191 191 191 F F F
120232024-02-20T22:46:29.025Z 23 460-479 109 109 109 191 191 191 F F F
120242024-02-20T22:46:29.026Z 24 480-499 111 111 111 194 194 194 F F F
120252024-02-20T22:46:29.026Z 25 500-519 110 110 110 192 192 192 F F F
120262024-02-20T22:46:29.026Z 26 520-539 108 108 108 187 187 187 F F F
120272024-02-20T22:46:29.026Z 27 540-559 102 102 102 178 178 178 F F F
120282024-02-20T22:46:29.026Z 28 560-579 108 108 108 188 188 188 F F F
120292024-02-20T22:46:29.026Z 29 580-599 105 105 105 181 181 181 F F F
120302024-02-20T22:46:29.026ZMax gen: 111, Max flush: 194
120312024-02-20T22:46:29.026Z
120322024-02-20T22:46:29.026ZBegin loop 56
120332024-02-20T22:46:29.026ZDownstairs to restart: 0
120342024-02-20T22:46:29.042Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.560998547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586}
120352024-02-20T22:46:29.050Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
120362024-02-20T22:46:29.050Z,"time":"2024-02-20T22:46:28.561380932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586}
120372024-02-20T22:46:29.050Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561415091Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586}
120382024-02-20T22:46:29.050Z{"msg":"Crucible b726cd3a-73ec-4544-883d-70df665aeddc has session id: 4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561451779Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120392024-02-20T22:46:29.050Z{"msg":"Upstairs opts: Upstairs UUID: b726cd3a-73ec-4544-883d-70df665aeddc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561491258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120402024-02-20T22:46:29.050Z{"msg":"Crucible stats registered with UUID: b726cd3a-73ec-4544-883d-70df665aeddc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561509767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120412024-02-20T22:46:29.050Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561523767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586}
120422024-02-20T22:46:29.050Z{"msg":"b726cd3a-73ec-4544-883d-70df665aeddc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561553926Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120432024-02-20T22:46:29.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561577745Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120442024-02-20T22:46:29.050Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561593504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120452024-02-20T22:46:29.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561607364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120462024-02-20T22:46:29.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561621323Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120472024-02-20T22:46:29.051Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561635523Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120482024-02-20T22:46:29.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561649232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120492024-02-20T22:46:29.051Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.56169735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120502024-02-20T22:46:29.051Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561732469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120512024-02-20T22:46:29.051Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.561748729Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120522024-02-20T22:46:29.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562020709Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120532024-02-20T22:46:29.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562072117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120542024-02-20T22:46:29.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562149064Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120552024-02-20T22:46:29.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562713414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120562024-02-20T22:46:29.051Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562749683Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120572024-02-20T22:46:29.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562766322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120582024-02-20T22:46:29.051Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562785632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120592024-02-20T22:46:29.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.562809201Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120602024-02-20T22:46:29.051Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.5628309Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120612024-02-20T22:46:29.051Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563351221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120622024-02-20T22:46:29.051Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563451828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120632024-02-20T22:46:29.051Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563503806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120642024-02-20T22:46:29.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563844474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120652024-02-20T22:46:29.051Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563903512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120662024-02-20T22:46:29.051Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563933681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120672024-02-20T22:46:29.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.56395436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120682024-02-20T22:46:29.051Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.563990179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120692024-02-20T22:46:29.051Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564017587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120702024-02-20T22:46:29.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564042577Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120712024-02-20T22:46:29.052Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564058216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120722024-02-20T22:46:29.052Z{"msg":"[0]R flush_numbers[0..12]: [193, 193, 185, 190, 193, 187, 194, 194, 186, 191, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564116894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120732024-02-20T22:46:29.052Z{"msg":"[0]R generation[0..12]: [110, 110, 106, 108, 110, 108, 111, 111, 107, 109, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564145843Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120742024-02-20T22:46:29.052Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564162142Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120752024-02-20T22:46:29.052Z{"msg":"[1]R flush_numbers[0..12]: [193, 193, 185, 190, 193, 187, 194, 194, 186, 191, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564177492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120762024-02-20T22:46:29.052Z{"msg":"[1]R generation[0..12]: [110, 110, 106, 108, 110, 108, 111, 111, 107, 109, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564198911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120772024-02-20T22:46:29.052Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564213451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120782024-02-20T22:46:29.052Z{"msg":"[2]R flush_numbers[0..12]: [193, 193, 185, 190, 193, 187, 194, 194, 186, 191, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.56422762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120792024-02-20T22:46:29.052Z{"msg":"[2]R generation[0..12]: [110, 110, 106, 108, 110, 108, 111, 111, 107, 109, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564242179Z","hostname":"Wait for a query_work_queue command to finish before sending IO
120802024-02-20T22:46:29.052Zip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120812024-02-20T22:46:29.052Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564261289Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120822024-02-20T22:46:29.052Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564282438Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120832024-02-20T22:46:29.052Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564296628Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120842024-02-20T22:46:29.052Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564309827Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120852024-02-20T22:46:29.052Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564330086Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120862024-02-20T22:46:29.052Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564343876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120872024-02-20T22:46:29.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564356255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120882024-02-20T22:46:29.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564376425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120892024-02-20T22:46:29.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564390094Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120902024-02-20T22:46:29.052Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564403694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120912024-02-20T22:46:29.052Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564423903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120922024-02-20T22:46:29.052Z{"msg":"b726cd3a-73ec-4544-883d-70df665aeddc is now active with session: 4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564437592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120932024-02-20T22:46:29.052Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564449692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586,"session_id":"4a9cb0df-adce-46cd-b0f1-5c8d3d8ee469"}
120942024-02-20T22:46:29.052Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:28.564462022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586}
120952024-02-20T22:46:29.052ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120962024-02-20T22:46:29.053ZLoading write count information from file "/tmp/test_repair_verify.data"
120972024-02-20T22:46:29.053ZRun Repair workload
120982024-02-20T22:46:29.053Z01/30 Write block 49 len 24576 data: 8 6 6 8 6 8
120992024-02-20T22:46:29.060Z02/30 Flush
121002024-02-20T22:46:29.060Z03/30 Write block 536 len 24576 data: 12 10 12 10 12 11
121012024-02-20T22:46:29.060Z04/30 Write block 413 len 20480 data: 15 15 14 12 10
121022024-02-20T22:46:29.060Z05/30 Flush
121032024-02-20T22:46:29.060Z06/30 Write block 577 len 24576 data: 10 12 14 15 12 12
121042024-02-20T22:46:29.060Z07/30 Write block 469 len 8192 data: 13 9
121052024-02-20T22:46:29.060Z08/30 Write block 414 len 32768 data: 16 15 13 11 11 14 14 10
121062024-02-20T22:46:29.060Z09/30 Read block 329 len 8192
121072024-02-20T22:46:29.060Z10/30 Read block 235 len 16384
121082024-02-20T22:46:29.064Z11/30 Read block 54 len 32768
121092024-02-20T22:46:29.064Z12/30 Read block 294 len 16384
121102024-02-20T22:46:29.064Z13/30 Write block 307 len 12288 data: 11 10 12
121112024-02-20T22:46:29.064Z14/30 Flush
121122024-02-20T22:46:29.069Z15/30 Write block 189 len 28672 data: 8 7 7 5 5 5 7
121132024-02-20T22:46:29.069Z16/30 Read block 550 len 8192
121142024-02-20T22:46:29.069Z17/30 Flush
121152024-02-20T22:46:29.074Z18/30 Write block 280 len 16384 data: 5 7 7 9
121162024-02-20T22:46:29.074Z19/30 Write block 322 len 36864 data: 13 9 6 11 12 11 11 10 9
121172024-02-20T22:46:29.074Z20/30 Write block 22 len 4096 data: 10
121182024-02-20T22:46:29.074Z21/30 Write block 14 len 20480 data: 5 6 11 11 12
121192024-02-20T22:46:29.074Z22/30 Flush
121202024-02-20T22:46:29.078Z23/30 Write block 159 len 32768 data: 9 9 7 6 5 5 7 8
121212024-02-20T22:46:29.078Z24/30 Read block 116 len 32768
121222024-02-20T22:46:29.078Z25/30 Read block 30 len 32768
121232024-02-20T22:46:29.083Z26/30 Read block 144 len 24576
121242024-02-20T22:46:29.083Z27/30 Read block 38 len 20480
121252024-02-20T22:46:29.083Z28/30 Write block 587 len 4096 data: 8
121262024-02-20T22:46:29.083Z29/30 Read block 377 len 4096
121272024-02-20T22:46:29.083Z30/30 Read block 125 len 16384
121282024-02-20T22:46:29.083Z----------------------------------------------------------------
121292024-02-20T22:46:29.083Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:25
121302024-02-20T22:46:29.083ZStates: Active Active Active
121312024-02-20T22:46:29.083ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121322024-02-20T22:46:29.083Z 6 Acked 1005 Write 6 Sent Done Done false
121332024-02-20T22:46:29.083Z 7 Acked 1006 Write 2 Sent Done Done false
121342024-02-20T22:46:29.083Z 8 Acked 1007 Write 8 Sent Done Done false
121352024-02-20T22:46:29.083Z 9 Acked 1008 Read 2 Sent Done Done false
121362024-02-20T22:46:29.083Z 10 Acked 1009 Read 4 Sent Done Done false
121372024-02-20T22:46:29.083Z 11 Acked 1010 Read 8 Sent Done Done false
121382024-02-20T22:46:29.084Z 12 Acked 1011 Read 4 Sent Done Done false
121392024-02-20T22:46:29.084Z 13 Acked 1012 Write 3 Sent Done Done false
121402024-02-20T22:46:29.084Z 14 Acked 1013 Flush 0 Sent Done Done false
121412024-02-20T22:46:29.084Z 15 Acked 1014 Write 7 Sent Done Done false
121422024-02-20T22:46:29.084Z 16 Acked 1015 Read 2 Sent Done Done false
121432024-02-20T22:46:29.084Z 17 Acked 1016 Flush 0 Sent Done Done false
121442024-02-20T22:46:29.084Z 18 Acked 1017 Write 4 Sent Done Done false
121452024-02-20T22:46:29.084Z 19 Acked 1018 Write 9 Sent Done Done false
121462024-02-20T22:46:29.084Z 20 Acked 1019 Write 1 Sent Done Done false
121472024-02-20T22:46:29.084Z 21 Acked 1020 Write 5 Sent Done Done false
121482024-02-20T22:46:29.084Z 22 Acked 1021 Flush 0 Sent Done Done false
121492024-02-20T22:46:29.084Z 23 Acked 1022 Write 8 Sent Done Done false
121502024-02-20T22:46:29.084Z 24 Acked 1023 Read 8 Sent Done Done false
121512024-02-20T22:46:29.084Z 25 Acked 1024 Read 8 Sent Done Done false
121522024-02-20T22:46:29.084Z 26 Acked 1025 Read 6 Sent Done Done false
121532024-02-20T22:46:29.084Z 27 Acked 1026 Read 5 Sent Done Done false
121542024-02-20T22:46:29.084Z 28 Acked 1027 Write 1 Sent Done Done false
121552024-02-20T22:46:29.089Z 29 Acked 1028 Read 1 Sent Done Done false
121562024-02-20T22:46:29.089Z 30 Acked 1029 Read 4 Sent Sent Done false
121572024-02-20T22:46:29.089Z STATES DS:0 DS:1 DS:2 TOTAL
121582024-02-20T22:46:29.089Z New 0 0 0 0
121592024-02-20T22:46:29.089Z Sent 25 1 0 26
121602024-02-20T22:46:29.089Z Done 0 24 25 49
121612024-02-20T22:46:29.089Z Skipped 0 0 0 0
121622024-02-20T22:46:29.089Z Error 0 0 0 0
121632024-02-20T22:46:29.089ZLast Flush: 1004 1021 1021
121642024-02-20T22:46:29.089ZDownstairs last five completed: 1004 1003 1002 1001 1000
121652024-02-20T22:46:29.089ZUpstairs last five completed: 30 29 28 27 26
121662024-02-20T22:46:29.089Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:28.601618342Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1586}
121672024-02-20T22:46:29.089ZWrote out file "/tmp/test_repair_verify.data"
121682024-02-20T22:46:29.090Z
121692024-02-20T22:46:29.094ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121702024-02-20T22:46:29.105ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121712024-02-20T22:46:29.105Z 0 000-019 110 112 112 193 199 199 F F F <---
121722024-02-20T22:46:29.105Z 1 020-039 110 112 112 193 199 199 F F F <---
121732024-02-20T22:46:29.105Z 2 040-059 112 112 112 195 195 195 F F F
121742024-02-20T22:46:29.105Z 3 060-079 108 108 108 190 190 190 F F F
121752024-02-20T22:46:29.105Z 4 080-099 110 110 110 193 193 193 F F F
121762024-02-20T22:46:29.105Z 5 100-119 108 108 108 187 187 187 F F F
121772024-02-20T22:46:29.105Z 6 120-139 111 111 111 194 194 194 F F F
121782024-02-20T22:46:29.105Z 7 140-159 111 111 111 194 194 194 F T T <---
121792024-02-20T22:46:29.110Z 8 160-179 107 107 107 186 186 186 F T T <---
121802024-02-20T22:46:29.110Z 9 180-199 109 112 112 191 198 198 F F F <---
121812024-02-20T22:46:29.110Z 10 200-219 106 106 106 182 182 182 F F F
121822024-02-20T22:46:29.110Z 11 220-239 110 110 110 192 192 192 F F F
121832024-02-20T22:46:29.110Z 12 240-259 110 110 110 192 192 192 F F F
121842024-02-20T22:46:29.110Z 13 260-279 107 107 107 186 186 186 F F F
121852024-02-20T22:46:29.110Z 14 280-299 110 112 112 192 199 199 F F F <---
121862024-02-20T22:46:29.110Z 15 300-319 111 112 112 194 197 197 F F F <---
121872024-02-20T22:46:29.110Z 16 320-339 109 112 112 191 199 199 F F F <---
121882024-02-20T22:46:29.110Z 17 340-359 110 110 110 192 192 192 F F F
121892024-02-20T22:46:29.110Z 18 360-379 110 110 110 192 192 192 F F F
121902024-02-20T22:46:29.110Z 19 380-399 110 110 110 192 192 192 F F F
121912024-02-20T22:46:29.110Z 20 400-419 112 112 112 196 197 197 F F F <---
121922024-02-20T22:46:29.110Z 21 420-439 110 112 112 193 197 197 F F F <---
121932024-02-20T22:46:29.110Z 22 440-459 109 109 109 191 191 191 F F F
121942024-02-20T22:46:29.110Z 23 460-479 109 112 112 191 197 197 F F F <---
121952024-02-20T22:46:29.110Z 24 480-499 111 111 111 194 194 194 F F F
121962024-02-20T22:46:29.110Z 25 500-519 110 110 110 192 192 192 F F F
121972024-02-20T22:46:29.110Z 26 520-539 112 112 112 196 196 196 F F F
121982024-02-20T22:46:29.110Z 27 540-559 112 112 112 196 196 196 F F F
121992024-02-20T22:46:29.110Z 28 560-579 108 112 112 188 197 197 F F F <---
122002024-02-20T22:46:29.110Z 29 580-599 105 112 112 181 197 197 F T T <---
122012024-02-20T22:46:29.111ZMax gen: 112, Max flush: 199
122022024-02-20T22:46:29.111ZError: Difference in extent metadata found!
122032024-02-20T22:46:29.111ZA Difference in extent metadata is expected here
122042024-02-20T22:46:29.111ZOn loop 56
122052024-02-20T22:46:29.111Z
122062024-02-20T22:46:29.115ZVerifying data now
122072024-02-20T22:46:33.752ZNo change after verify
122082024-02-20T22:46:33.753ZLoop: 56 Downstairs dump after verify (and repair):
122092024-02-20T22:46:33.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122102024-02-20T22:46:33.767Z 0 000-019 112 112 112 199 199 199 F F F
122112024-02-20T22:46:33.767Z 1 020-039 112 112 112 199 199 199 F F F
122122024-02-20T22:46:33.767Z 2 040-059 112 112 112 195 195 195 F F F
122132024-02-20T22:46:33.767Z 3 060-079 108 108 108 190 190 190 F F F
122142024-02-20T22:46:33.767Z 4 080-099 110 110 110 193 193 193 F F F
122152024-02-20T22:46:33.767Z 5 100-119 108 108 108 187 187 187 F F F
122162024-02-20T22:46:33.767Z 6 120-139 111 111 111 194 194 194 F F F
122172024-02-20T22:46:33.767Z 7 140-159 113 113 113 200 200 200 F F F
122182024-02-20T22:46:33.767Z 8 160-179 113 113 113 200 200 200 F F F
122192024-02-20T22:46:33.767Z 9 180-199 112 112 112 198 198 198 F F F
122202024-02-20T22:46:33.767Z 10 200-219 106 106 106 182 182 182 F F F
122212024-02-20T22:46:33.767Z 11 220-239 110 110 110 192 192 192 F F F
122222024-02-20T22:46:33.767Z 12 240-259 110 110 110 192 192 192 F F F
122232024-02-20T22:46:33.767Z 13 260-279 107 107 107 186 186 186 F F F
122242024-02-20T22:46:33.773Z 14 280-299 112 112 112 199 199 199 F F F
122252024-02-20T22:46:33.773Z 15 300-319 112 112 112 197 197 197 F F F
122262024-02-20T22:46:33.773Z 16 320-339 112 112 112 199 199 199 F F F
122272024-02-20T22:46:33.774Z 17 340-359 110 110 110 192 192 192 F F F
122282024-02-20T22:46:33.774Z 18 360-379 110 110 110 192 192 192 F F F
122292024-02-20T22:46:33.774Z 19 380-399 110 110 110 192 192 192 F F F
122302024-02-20T22:46:33.774Z 20 400-419 112 112 112 197 197 197 F F F
122312024-02-20T22:46:33.774Z 21 420-439 112 112 112 197 197 197 F F F
122322024-02-20T22:46:33.774Z 22 440-459 109 109 109 191 191 191 F F F
122332024-02-20T22:46:33.774Z 23 460-479 112 112 112 197 197 197 F F F
122342024-02-20T22:46:33.774Z 24 480-499 111 111 111 194 194 194 F F F
122352024-02-20T22:46:33.775Z 25 500-519 110 110 110 192 192 192 F F F
122362024-02-20T22:46:33.775Z 26 520-539 112 112 112 196 196 196 F F F
122372024-02-20T22:46:33.775Z 27 540-559 112 112 112 196 196 196 F F F
122382024-02-20T22:46:33.775Z 28 560-579 112 112 112 197 197 197 F F F
122392024-02-20T22:46:33.775Z 29 580-599 113 113 113 200 200 200 F F F
122402024-02-20T22:46:33.775ZMax gen: 113, Max flush: 200
122412024-02-20T22:46:33.775Z
122422024-02-20T22:46:33.775ZBegin loop 57
122432024-02-20T22:46:33.775ZDownstairs to restart: 2
122442024-02-20T22:46:33.792Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305085079Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594}
122452024-02-20T22:46:33.803Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305439066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594}
122462024-02-20T22:46:33.803ZCrucible runtime is spawned
122472024-02-20T22:46:33.803Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305541512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594}
122482024-02-20T22:46:33.803Z{"msg":"Crucible 61ad26dc-66da-42eb-a8e3-961db33df433 has session id: c44696c2-8862-45d9-9f64-d8c8080b580e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305578801Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122492024-02-20T22:46:33.803Z{"msg":"Upstairs opts: Upstairs UUID: 61ad26dc-66da-42eb-a8e3-961db33df433, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30561411Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122502024-02-20T22:46:33.803Z{"msg":"Crucible stats registered with UUID: 61ad26dc-66da-42eb-a8e3-961db33df433","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305640549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122512024-02-20T22:46:33.803Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305662798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594}
122522024-02-20T22:46:33.803Z{"msg":"61ad26dc-66da-42eb-a8e3-961db33df433 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305720256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122532024-02-20T22:46:33.803Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305748025Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122542024-02-20T22:46:33.803Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305763844Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122552024-02-20T22:46:33.804Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305786004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122562024-02-20T22:46:33.804Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305816183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122572024-02-20T22:46:33.804Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305846222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122582024-02-20T22:46:33.804Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30587085Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122592024-02-20T22:46:33.804Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30589874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122602024-02-20T22:46:33.804Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305929759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122612024-02-20T22:46:33.804Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.305961007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122622024-02-20T22:46:33.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.306130611Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122632024-02-20T22:46:33.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.306185789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122642024-02-20T22:46:33.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.306211508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122652024-02-20T22:46:33.804Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.306902814Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122662024-02-20T22:46:33.804Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.306944762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122672024-02-20T22:46:33.804Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30699476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122682024-02-20T22:46:33.804Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.307050808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122692024-02-20T22:46:33.804Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.307084347Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122702024-02-20T22:46:33.804Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.307109426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122712024-02-20T22:46:33.804Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.307535021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122722024-02-20T22:46:33.804Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.3075763Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122732024-02-20T22:46:33.804Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.307649357Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122742024-02-20T22:46:33.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308049903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122752024-02-20T22:46:33.804Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308091651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122762024-02-20T22:46:33.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30810821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122772024-02-20T22:46:33.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308189147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122782024-02-20T22:46:33.804Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308258215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122792024-02-20T22:46:33.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308290274Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122802024-02-20T22:46:33.805Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308305223Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122812024-02-20T22:46:33.805Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308326923Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122822024-02-20T22:46:33.805Z{"msg":"[0]R flush_numbers[0..12]: [199, 199, 195, 190, 193, 187, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308342162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122832024-02-20T22:46:33.805Z{"msg":"[0]R generation[0..12]: [112, 112, 112, 108, 110, 108, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308364051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122842024-02-20T22:46:33.805Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308378951Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122852024-02-20T22:46:33.805ZWait for a query_work_queue command to finish before sending IO
122862024-02-20T22:46:33.805Z{"msg":"[1]R flush_numbers[0..12]: [199, 199, 195, 190, 193, 187, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30840408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122872024-02-20T22:46:33.805Z{"msg":"[1]R generation[0..12]: [112, 112, 112, 108, 110, 108, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308419019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122882024-02-20T22:46:33.805Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308433349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122892024-02-20T22:46:33.805Z{"msg":"[2]R flush_numbers[0..12]: [199, 199, 195, 190, 193, 187, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308454078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122902024-02-20T22:46:33.805Z{"msg":"[2]R generation[0..12]: [112, 112, 112, 108, 110, 108, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308468008Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122912024-02-20T22:46:33.805Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308493137Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122922024-02-20T22:46:33.805Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308511466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122932024-02-20T22:46:33.805Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308529905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122942024-02-20T22:46:33.805Z{"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308545725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122952024-02-20T22:46:33.805Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308559124Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122962024-02-20T22:46:33.805Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308581464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122972024-02-20T22:46:33.805Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308603753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122982024-02-20T22:46:33.805Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308623862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
122992024-02-20T22:46:33.805Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308654711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
123002024-02-20T22:46:33.805Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.30867109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
123012024-02-20T22:46:33.805Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
123022024-02-20T22:46:33.805Z{"msg":"61ad26dc-66da-42eb-a8e3-961db33df433 is now active with session: c44696c2-8862-45d9-9f64-d8c8080b580e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308730628Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
123032024-02-20T22:46:33.805Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308761907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594,"session_id":"c44696c2-8862-45d9-9f64-d8c8080b580e"}
123042024-02-20T22:46:33.805ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123052024-02-20T22:46:33.806Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:33.308799736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594}
123062024-02-20T22:46:33.806ZLoading write count information from file "/tmp/test_repair_verify.data"
123072024-02-20T22:46:33.806ZRun Repair workload
123082024-02-20T22:46:33.806Z01/30 Write block 478 len 4096 data: 15
123092024-02-20T22:46:33.806Z02/30 Read block 218 len 36864
123102024-02-20T22:46:33.806Z03/30 Read block 259 len 28672
123112024-02-20T22:46:33.806Z04/30 Read block 298 len 16384
123122024-02-20T22:46:33.806Z05/30 Read block 138 len 24576
123132024-02-20T22:46:33.806Z06/30 Write block 110 len 4096 data: 10
123142024-02-20T22:46:33.806Z07/30 Read block 560 len 16384
123152024-02-20T22:46:33.806Z08/30 Read block 448 len 28672
123162024-02-20T22:46:33.806Z09/30 Write block 441 len 8192 data: 13 11
123172024-02-20T22:46:33.806Z10/30 Read block 372 len 12288
123182024-02-20T22:46:33.811Z11/30 Write block 97 len 24576 data: 6 3 8 7 6 5
123192024-02-20T22:46:33.811Z12/30 Write block 303 len 4096 data: 14
123202024-02-20T22:46:33.811Z13/30 Read block 1 len 4096
123212024-02-20T22:46:33.811Z14/30 Write block 535 len 20480 data: 12 13 11 13 11
123222024-02-20T22:46:33.811Z15/30 Write block 593 len 24576 data: 11 12 12 12 11 9
123232024-02-20T22:46:33.811Z16/30 Read block 278 len 40960
123242024-02-20T22:46:33.811Z17/30 Read block 230 len 16384
123252024-02-20T22:46:33.811Z18/30 Write block 343 len 32768 data: 11 7 7 7 9 10 8 8
123262024-02-20T22:46:33.811Z19/30 Read block 166 len 12288
123272024-02-20T22:46:33.811Z20/30 Write block 591 len 4096 data: 13
123282024-02-20T22:46:33.811Z21/30 Read block 471 len 36864
123292024-02-20T22:46:33.812Z22/30 Write block 246 len 12288 data: 4 8 7
123302024-02-20T22:46:33.812Z23/30 Read block 224 len 8192
123312024-02-20T22:46:33.812Z24/30 Write block 78 len 12288 data: 7 6 6
123322024-02-20T22:46:33.812Z25/30 Write block 33 len 24576 data: 16 13 13 12 11 11
123332024-02-20T22:46:33.812Z26/30 Read block 367 len 16384
123342024-02-20T22:46:33.816Z27/30 Write block 30 len 24576 data: 16 17 17 17 14 14
123352024-02-20T22:46:33.816Z28/30 Read block 502 len 4096
123362024-02-20T22:46:33.816Z29/30 Read block 399 len 4096
123372024-02-20T22:46:33.816Z30/30 Write block 371 len 8192 data: 16 14
123382024-02-20T22:46:33.816Z----------------------------------------------------------------
123392024-02-20T22:46:33.816Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123402024-02-20T22:46:33.816ZStates: Active Active Active
123412024-02-20T22:46:33.816ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123422024-02-20T22:46:33.816Z 1 Acked 1000 Write 1 Done Done Sent false
123432024-02-20T22:46:33.816Z 2 Acked 1001 Read 9 Done Done Sent false
123442024-02-20T22:46:33.816Z 3 Acked 1002 Read 7 Done Done Sent false
123452024-02-20T22:46:33.816Z 4 Acked 1003 Read 4 Done Done Sent false
123462024-02-20T22:46:33.816Z 5 Acked 1004 Read 6 Done Done Sent false
123472024-02-20T22:46:33.817Z 6 Acked 1005 Write 1 Done Done Sent false
123482024-02-20T22:46:33.817Z 7 Acked 1006 Read 4 Done Done Sent false
123492024-02-20T22:46:33.817Z 8 Acked 1007 Read 7 Done Done Sent false
123502024-02-20T22:46:33.817Z 9 Acked 1008 Write 2 Done Done Sent false
123512024-02-20T22:46:33.817Z 10 Acked 1009 Read 3 Done Done Sent false
123522024-02-20T22:46:33.817Z 11 Acked 1010 Write 6 Done Done Sent false
123532024-02-20T22:46:33.817Z 12 Acked 1011 Write 1 Done Done Sent false
123542024-02-20T22:46:33.817Z 13 Acked 1012 Read 1 Done Done Sent false
123552024-02-20T22:46:33.817Z 14 Acked 1013 Write 5 Done Done Sent false
123562024-02-20T22:46:33.817Z 15 Acked 1014 Write 6 Done Done Sent false
123572024-02-20T22:46:33.817Z 16 Acked 1015 Read 10 Done Done Sent false
123582024-02-20T22:46:33.817Z 17 Acked 1016 Read 4 Done Done Sent false
123592024-02-20T22:46:33.817Z 18 Acked 1017 Write 8 Done Done Sent false
123602024-02-20T22:46:33.817Z 19 Acked 1018 Read 3 Done Done Sent false
123612024-02-20T22:46:33.817Z 20 Acked 1019 Write 1 Done Done Sent false
123622024-02-20T22:46:33.817Z 21 Acked 1020 Read 9 Done Done Sent false
123632024-02-20T22:46:33.817Z 22 Acked 1021 Write 3 Done Done Sent false
123642024-02-20T22:46:33.817Z 23 Acked 1022 Read 2 Done Done Sent false
123652024-02-20T22:46:33.817Z 24 Acked 1023 Write 3 Done Done Sent false
123662024-02-20T22:46:33.817Z 25 Acked 1024 Write 6 Done Done Sent false
123672024-02-20T22:46:33.817Z 26 Acked 1025 Read 4 Done Done Sent false
123682024-02-20T22:46:33.817Z 27 Acked 1026 Write 6 Done Done Sent false
123692024-02-20T22:46:33.817Z 28 Acked 1027 Read 1 Done Done Sent false
123702024-02-20T22:46:33.817Z 29 Acked 1028 Read 1 Done Done Sent false
123712024-02-20T22:46:33.817Z 30 Acked 1029 Write 2 Sent Sent Sent false
123722024-02-20T22:46:33.818Z STATES DS:0 DS:1 DS:2 TOTAL
123732024-02-20T22:46:33.818Z New 0 0 0 0
123742024-02-20T22:46:33.818Z Sent 1 1 30 32
123752024-02-20T22:46:33.818Z Done 29 29 0 58
123762024-02-20T22:46:33.818Z Skipped 0 0 0 0
123772024-02-20T22:46:33.818Z Error 0 0 0 0
123782024-02-20T22:46:33.818ZLast Flush: 0 0 0
123792024-02-20T22:46:33.818ZDownstairs last five completed:
123802024-02-20T22:46:33.818ZUpstairs last five completed: 30 29 28 27 26
123812024-02-20T22:46:33.818Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:33.332671571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1594}
123822024-02-20T22:46:33.818ZWrote out file "/tmp/test_repair_verify.data"
123832024-02-20T22:46:33.823Z
123842024-02-20T22:46:33.823ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123852024-02-20T22:46:33.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123862024-02-20T22:46:33.840Z 0 000-019 112 112 112 199 199 199 F F F
123872024-02-20T22:46:33.840Z 1 020-039 112 112 112 199 199 199 T T F <---
123882024-02-20T22:46:33.840Z 2 040-059 112 112 112 195 195 195 F F F
123892024-02-20T22:46:33.840Z 3 060-079 108 108 108 190 190 190 T T F <---
123902024-02-20T22:46:33.840Z 4 080-099 110 110 110 193 193 193 T T F <---
123912024-02-20T22:46:33.840Z 5 100-119 108 108 108 187 187 187 T T F <---
123922024-02-20T22:46:33.840Z 6 120-139 111 111 111 194 194 194 F F F
123932024-02-20T22:46:33.840Z 7 140-159 113 113 113 200 200 200 F F F
123942024-02-20T22:46:33.840Z 8 160-179 113 113 113 200 200 200 F F F
123952024-02-20T22:46:33.840Z 9 180-199 112 112 112 198 198 198 F F F
123962024-02-20T22:46:33.845Z 10 200-219 106 106 106 182 182 182 F F F
123972024-02-20T22:46:33.845Z 11 220-239 110 110 110 192 192 192 F F F
123982024-02-20T22:46:33.845Z 12 240-259 110 110 110 192 192 192 T T F <---
123992024-02-20T22:46:33.845Z 13 260-279 107 107 107 186 186 186 F F F
124002024-02-20T22:46:33.845Z 14 280-299 112 112 112 199 199 199 F F F
124012024-02-20T22:46:33.845Z 15 300-319 112 112 112 197 197 197 T T F <---
124022024-02-20T22:46:33.845Z 16 320-339 112 112 112 199 199 199 F F F
124032024-02-20T22:46:33.845Z 17 340-359 110 110 110 192 192 192 T T F <---
124042024-02-20T22:46:33.845Z 18 360-379 110 110 110 192 192 192 T T F <---
124052024-02-20T22:46:33.845Z 19 380-399 110 110 110 192 192 192 F F F
124062024-02-20T22:46:33.845Z 20 400-419 112 112 112 197 197 197 F F F
124072024-02-20T22:46:33.845Z 21 420-439 112 112 112 197 197 197 F F F
124082024-02-20T22:46:33.845Z 22 440-459 109 109 109 191 191 191 T T F <---
124092024-02-20T22:46:33.845Z 23 460-479 112 112 112 197 197 197 T T F <---
124102024-02-20T22:46:33.845Z 24 480-499 111 111 111 194 194 194 F F F
124112024-02-20T22:46:33.845Z 25 500-519 110 110 110 192 192 192 F F F
124122024-02-20T22:46:33.845Z 26 520-539 112 112 112 196 196 196 T T F <---
124132024-02-20T22:46:33.845Z 27 540-559 112 112 112 196 196 196 F F F
124142024-02-20T22:46:33.845Z 28 560-579 112 112 112 197 197 197 F F F
124152024-02-20T22:46:33.845Z 29 580-599 113 113 113 200 200 200 T T F <---
124162024-02-20T22:46:33.845ZMax gen: 113, Max flush: 200
124172024-02-20T22:46:33.845ZError: Difference in extent metadata found!
124182024-02-20T22:46:33.846ZA Difference in extent metadata is expected here
124192024-02-20T22:46:33.846ZOn loop 57
124202024-02-20T22:46:33.846Z
124212024-02-20T22:46:33.849ZVerifying data now
124222024-02-20T22:46:38.461ZNo change after verify
124232024-02-20T22:46:38.461ZLoop: 57 Downstairs dump after verify (and repair):
124242024-02-20T22:46:38.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124252024-02-20T22:46:38.475Z 0 000-019 112 112 112 199 199 199 F F F
124262024-02-20T22:46:38.475Z 1 020-039 114 114 114 201 201 201 F F F
124272024-02-20T22:46:38.475Z 2 040-059 112 112 112 195 195 195 F F F
124282024-02-20T22:46:38.476Z 3 060-079 114 114 114 201 201 201 F F F
124292024-02-20T22:46:38.476Z 4 080-099 114 114 114 201 201 201 F F F
124302024-02-20T22:46:38.476Z 5 100-119 114 114 114 201 201 201 F F F
124312024-02-20T22:46:38.480Z 6 120-139 111 111 111 194 194 194 F F F
124322024-02-20T22:46:38.480Z 7 140-159 113 113 113 200 200 200 F F F
124332024-02-20T22:46:38.480Z 8 160-179 113 113 113 200 200 200 F F F
124342024-02-20T22:46:38.480Z 9 180-199 112 112 112 198 198 198 F F F
124352024-02-20T22:46:38.480Z 10 200-219 106 106 106 182 182 182 F F F
124362024-02-20T22:46:38.480Z 11 220-239 110 110 110 192 192 192 F F F
124372024-02-20T22:46:38.480Z 12 240-259 114 114 114 201 201 201 F F F
124382024-02-20T22:46:38.480Z 13 260-279 107 107 107 186 186 186 F F F
124392024-02-20T22:46:38.481Z 14 280-299 112 112 112 199 199 199 F F F
124402024-02-20T22:46:38.481Z 15 300-319 114 114 114 201 201 201 F F F
124412024-02-20T22:46:38.481Z 16 320-339 112 112 112 199 199 199 F F F
124422024-02-20T22:46:38.481Z 17 340-359 114 114 114 201 201 201 F F F
124432024-02-20T22:46:38.481Z 18 360-379 114 114 114 201 201 201 F F F
124442024-02-20T22:46:38.481Z 19 380-399 110 110 110 192 192 192 F F F
124452024-02-20T22:46:38.481Z 20 400-419 112 112 112 197 197 197 F F F
124462024-02-20T22:46:38.481Z 21 420-439 112 112 112 197 197 197 F F F
124472024-02-20T22:46:38.481Z 22 440-459 114 114 114 201 201 201 F F F
124482024-02-20T22:46:38.481Z 23 460-479 114 114 114 201 201 201 F F F
124492024-02-20T22:46:38.481Z 24 480-499 111 111 111 194 194 194 F F F
124502024-02-20T22:46:38.481Z 25 500-519 110 110 110 192 192 192 F F F
124512024-02-20T22:46:38.481Z 26 520-539 114 114 114 201 201 201 F F F
124522024-02-20T22:46:38.481Z 27 540-559 112 112 112 196 196 196 F F F
124532024-02-20T22:46:38.481Z 28 560-579 112 112 112 197 197 197 F F F
124542024-02-20T22:46:38.481Z 29 580-599 114 114 114 201 201 201 F F F
124552024-02-20T22:46:38.481ZMax gen: 114, Max flush: 201
124562024-02-20T22:46:38.481Z
124572024-02-20T22:46:38.481ZBegin loop 58
124582024-02-20T22:46:38.481ZDownstairs to restart: 2
124592024-02-20T22:46:38.498Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016268368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602}
124602024-02-20T22:46:38.498Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016557918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602}
124612024-02-20T22:46:38.508Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016600997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602}
124622024-02-20T22:46:38.508ZCrucible runtime is spawned
124632024-02-20T22:46:38.508Z{"msg":"Crucible 7afa8f21-8e33-4f13-9c45-740137a72301 has session id: 0d16bf7b-4a91-4d60-b323-88f060d86b3f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016648665Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124642024-02-20T22:46:38.508Z{"msg":"Upstairs opts: Upstairs UUID: 7afa8f21-8e33-4f13-9c45-740137a72301, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016737442Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124652024-02-20T22:46:38.508Z{"msg":"Crucible stats registered with UUID: 7afa8f21-8e33-4f13-9c45-740137a72301","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01678688Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124662024-02-20T22:46:38.508Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016845348Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602}
124672024-02-20T22:46:38.508Z{"msg":"7afa8f21-8e33-4f13-9c45-740137a72301 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016889076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124682024-02-20T22:46:38.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016911045Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124692024-02-20T22:46:38.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016941134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124702024-02-20T22:46:38.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.016991492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124712024-02-20T22:46:38.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017031391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124722024-02-20T22:46:38.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01705735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124732024-02-20T22:46:38.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017118938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124742024-02-20T22:46:38.509Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017196265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124752024-02-20T22:46:38.509Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017254253Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124762024-02-20T22:46:38.509Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017307461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124772024-02-20T22:46:38.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017368569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124782024-02-20T22:46:38.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017425087Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124792024-02-20T22:46:38.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017518534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124802024-02-20T22:46:38.509Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01789458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124812024-02-20T22:46:38.509Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.017979537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124822024-02-20T22:46:38.509Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.018049075Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124832024-02-20T22:46:38.509Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.018076264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124842024-02-20T22:46:38.509Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.018107872Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124852024-02-20T22:46:38.509Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01819606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124862024-02-20T22:46:38.509Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01846415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124872024-02-20T22:46:38.509Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.018645353Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124882024-02-20T22:46:38.509Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.018903624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124892024-02-20T22:46:38.509Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019058129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124902024-02-20T22:46:38.509Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019104557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124912024-02-20T22:46:38.509Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019136996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124922024-02-20T22:46:38.509Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019236572Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124932024-02-20T22:46:38.509Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01929674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124942024-02-20T22:46:38.510Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019347098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124952024-02-20T22:46:38.510Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019373097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124962024-02-20T22:46:38.510Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019404176Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124972024-02-20T22:46:38.510Z{"msg":"[0]R flush_numbers[0..12]: [199, 201, 195, 201, 201, 201, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019430445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124982024-02-20T22:46:38.510Z{"msg":"[0]R generation[0..12]: [112, 114, 112, 114, 114, 114, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019460414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
124992024-02-20T22:46:38.510Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019487523Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125002024-02-20T22:46:38.510Z{"msg":"[1]R flush_numbers[0..12]: [199, 201, 195, 201, 201, 201, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019520932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125012024-02-20T22:46:38.510Z{"msg":"[1]R generation[0..12]: [112, 114, 112, 114, 114, 114, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019547431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
125022024-02-20T22:46:38.510Z,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125032024-02-20T22:46:38.510Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01958576Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125042024-02-20T22:46:38.510Z{"msg":"[2]R flush_numbers[0..12]: [199, 201, 195, 201, 201, 201, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019620389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125052024-02-20T22:46:38.510Z{"msg":"[2]R generation[0..12]: [112, 114, 112, 114, 114, 114, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019646437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125062024-02-20T22:46:38.510Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019679526Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125072024-02-20T22:46:38.510Z{"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019721355Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125082024-02-20T22:46:38.510Z{"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019746934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125092024-02-20T22:46:38.510Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019777633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125102024-02-20T22:46:38.510Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019801312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125112024-02-20T22:46:38.510Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019828921Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125122024-02-20T22:46:38.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.01985147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125132024-02-20T22:46:38.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019883449Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125142024-02-20T22:46:38.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019907768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125152024-02-20T22:46:38.510Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019940927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125162024-02-20T22:46:38.510Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.019965026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125172024-02-20T22:46:38.510Z{"msg":"7afa8f21-8e33-4f13-9c45-740137a72301 is now active with session: 0d16bf7b-4a91-4d60-b323-88f060d86b3f","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125182024-02-20T22:46:38.510Z,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.020011484Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125192024-02-20T22:46:38.511Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.020045713Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602,"session_id":"0d16bf7b-4a91-4d60-b323-88f060d86b3f"}
125202024-02-20T22:46:38.511Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.020069382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602}
125212024-02-20T22:46:38.511ZLoading write count information from file "/tmp/test_repair_verify.data"
125222024-02-20T22:46:38.511ZRun Repair workload
125232024-02-20T22:46:38.511Z01/30 Write block 54 len 4096 data: 9
125242024-02-20T22:46:38.516Z02/30 Read block 85 len 8192
125252024-02-20T22:46:38.516Z03/30 Read block 447 len 4096
125262024-02-20T22:46:38.516Z04/30 Read block 388 len 24576
125272024-02-20T22:46:38.516Z05/30 Read block 115 len 16384
125282024-02-20T22:46:38.516Z06/30 Read block 192 len 16384
125292024-02-20T22:46:38.516Z07/30 Flush
125302024-02-20T22:46:38.516Z08/30 Write block 250 len 24576 data: 6 5 5 8 7 5
125312024-02-20T22:46:38.516Z09/30 Write block 396 len 36864 data: 12 14 8 10 10 8 10 11 11
125322024-02-20T22:46:38.516Z10/30 Read block 471 len 8192
125332024-02-20T22:46:38.516Z11/30 Read block 150 len 12288
125342024-02-20T22:46:38.516Z12/30 Read block 455 len 4096
125352024-02-20T22:46:38.516Z13/30 Read block 520 len 8192
125362024-02-20T22:46:38.516Z14/30 Read block 250 len 40960
125372024-02-20T22:46:38.516Z15/30 Write block 16 len 16384 data: 12 12 13 12
125382024-02-20T22:46:38.521Z16/30 Write block 520 len 36864 data: 9 10 9 7 5 6 9 10 8
125392024-02-20T22:46:38.521Z17/30 Read block 475 len 40960
125402024-02-20T22:46:38.521Z18/30 Read block 153 len 36864
125412024-02-20T22:46:38.521Z19/30 Read block 272 len 24576
125422024-02-20T22:46:38.521Z20/30 Write block 511 len 12288 data: 11 8 9
125432024-02-20T22:46:38.521Z21/30 Read block 509 len 20480
125442024-02-20T22:46:38.527Z22/30 Read block 186 len 8192
125452024-02-20T22:46:38.527Z23/30 Write block 538 len 36864 data: 14 12 13 12 9 10 9 9 9
125462024-02-20T22:46:38.527Z24/30 Read block 44 len 24576
125472024-02-20T22:46:38.527Z25/30 Read block 551 len 24576
125482024-02-20T22:46:38.527Z26/30 Read block 545 len 24576
125492024-02-20T22:46:38.527Z27/30 Write block 344 len 40960 data: 8 8 8 10 11 9 9 7 5 5
125502024-02-20T22:46:38.533Z28/30 Read block 78 len 8192
125512024-02-20T22:46:38.533Z29/30 Write block 59 len 36864 data: 9 6 5 6 7 7 7 6 10
125522024-02-20T22:46:38.533Z30/30 Read block 212 len 40960
125532024-02-20T22:46:38.533Z----------------------------------------------------------------
125542024-02-20T22:46:38.533Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:23
125552024-02-20T22:46:38.533ZStates: Active Active Active
125562024-02-20T22:46:38.533ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125572024-02-20T22:46:38.533Z 8 Acked 1007 Write 6 Done Done Sent false
125582024-02-20T22:46:38.533Z 9 Acked 1008 Write 9 Done Done Sent false
125592024-02-20T22:46:38.533Z 10 Acked 1009 Read 2 Done Done Sent false
125602024-02-20T22:46:38.533Z 11 Acked 1010 Read 3 Done Done Sent false
125612024-02-20T22:46:38.533Z 12 Acked 1011 Read 1 Done Done Sent false
125622024-02-20T22:46:38.533Z 13 Acked 1012 Read 2 Done Done Sent false
125632024-02-20T22:46:38.533Z 14 Acked 1013 Read 10 Done Done Sent false
125642024-02-20T22:46:38.533Z 15 Acked 1014 Write 4 Done Done Sent false
125652024-02-20T22:46:38.534Z 16 Acked 1015 Write 9 Done Done Sent false
125662024-02-20T22:46:38.534Z 17 Acked 1016 Read 10 Done Done Sent false
125672024-02-20T22:46:38.534Z 18 Acked 1017 Read 9 Done Done Sent false
125682024-02-20T22:46:38.534Z 19 Acked 1018 Read 6 Done Done Sent false
125692024-02-20T22:46:38.534Z 20 Acked 1019 Write 3 Done Done Sent false
125702024-02-20T22:46:38.534Z 21 Acked 1020 Read 5 Done Done Sent false
125712024-02-20T22:46:38.534Z 22 Acked 1021 Read 2 Done Done Sent false
125722024-02-20T22:46:38.534Z 23 Acked 1022 Write 9 Done Done Sent false
125732024-02-20T22:46:38.534Z 24 Acked 1023 Read 6 Done Done Sent false
125742024-02-20T22:46:38.534Z 25 Acked 1024 Read 6 Done Done Sent false
125752024-02-20T22:46:38.534Z 26 Acked 1025 Read 6 Done Done Sent false
125762024-02-20T22:46:38.534Z 27 Acked 1026 Write 10 Done Done Sent false
125772024-02-20T22:46:38.534Z 28 Acked 1027 Read 2 Done Done Sent false
125782024-02-20T22:46:38.534Z 29 Acked 1028 Write 9 Done Done Sent false
125792024-02-20T22:46:38.534Z 30 Acked 1029 Read 10 Done Sent Sent false
125802024-02-20T22:46:38.534Z STATES DS:0 DS:1 DS:2 TOTAL
125812024-02-20T22:46:38.534Z New 0 0 0 0
125822024-02-20T22:46:38.534Z Sent 0 1 23 24
125832024-02-20T22:46:38.534Z Done 23 22 0 45
125842024-02-20T22:46:38.534Z Skipped 0 0 0 0
125852024-02-20T22:46:38.534Z Error 0 0 0 0
125862024-02-20T22:46:38.534ZLast Flush: 1006 1006 1006
125872024-02-20T22:46:38.534ZDownstairs last five completed: 1006 1005 1004 1003 1002
125882024-02-20T22:46:38.534ZUpstairs last five completed: 30 29 28 27 26
125892024-02-20T22:46:38.534Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:38.048783485Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1602}
125902024-02-20T22:46:38.535ZWrote out file "/tmp/test_repair_verify.data"
125912024-02-20T22:46:38.539Z
125922024-02-20T22:46:38.539ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125932024-02-20T22:46:38.556ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125942024-02-20T22:46:38.556Z 0 000-019 112 112 112 199 199 199 T T F <---
125952024-02-20T22:46:38.556Z 1 020-039 114 114 114 201 201 201 F F F
125962024-02-20T22:46:38.556Z 2 040-059 116 116 116 202 202 202 T T F <---
125972024-02-20T22:46:38.556Z 3 060-079 114 114 114 201 201 201 T T F <---
125982024-02-20T22:46:38.556Z 4 080-099 114 114 114 201 201 201 F F F
125992024-02-20T22:46:38.556Z 5 100-119 114 114 114 201 201 201 F F F
126002024-02-20T22:46:38.556Z 6 120-139 111 111 111 194 194 194 F F F
126012024-02-20T22:46:38.556Z 7 140-159 113 113 113 200 200 200 F F F
126022024-02-20T22:46:38.556Z 8 160-179 113 113 113 200 200 200 F F F
126032024-02-20T22:46:38.561Z 9 180-199 112 112 112 198 198 198 F F F
126042024-02-20T22:46:38.561Z 10 200-219 106 106 106 182 182 182 F F F
126052024-02-20T22:46:38.561Z 11 220-239 110 110 110 192 192 192 F F F
126062024-02-20T22:46:38.561Z 12 240-259 114 114 114 201 201 201 T T F <---
126072024-02-20T22:46:38.561Z 13 260-279 107 107 107 186 186 186 F F F
126082024-02-20T22:46:38.561Z 14 280-299 112 112 112 199 199 199 F F F
126092024-02-20T22:46:38.561Z 15 300-319 114 114 114 201 201 201 F F F
126102024-02-20T22:46:38.561Z 16 320-339 112 112 112 199 199 199 F F F
126112024-02-20T22:46:38.561Z 17 340-359 114 114 114 201 201 201 T T F <---
126122024-02-20T22:46:38.561Z 18 360-379 114 114 114 201 201 201 F F F
126132024-02-20T22:46:38.561Z 19 380-399 110 110 110 192 192 192 T T F <---
126142024-02-20T22:46:38.561Z 20 400-419 112 112 112 197 197 197 T T F <---
126152024-02-20T22:46:38.561Z 21 420-439 112 112 112 197 197 197 F F F
126162024-02-20T22:46:38.561Z 22 440-459 114 114 114 201 201 201 F F F
126172024-02-20T22:46:38.561Z 23 460-479 114 114 114 201 201 201 F F F
126182024-02-20T22:46:38.561Z 24 480-499 111 111 111 194 194 194 F F F
126192024-02-20T22:46:38.562Z 25 500-519 110 110 110 192 192 192 T T F <---
126202024-02-20T22:46:38.562Z 26 520-539 114 114 114 201 201 201 T T F <---
126212024-02-20T22:46:38.562Z 27 540-559 112 112 112 196 196 196 T T F <---
126222024-02-20T22:46:38.562Z 28 560-579 112 112 112 197 197 197 F F F
126232024-02-20T22:46:38.562Z 29 580-599 114 114 114 201 201 201 F F F
126242024-02-20T22:46:38.562ZMax gen: 116, Max flush: 202
126252024-02-20T22:46:38.562ZError: Difference in extent metadata found!
126262024-02-20T22:46:38.562ZA Difference in extent metadata is expected here
126272024-02-20T22:46:38.562ZOn loop 58
126282024-02-20T22:46:38.562Z
126292024-02-20T22:46:38.566ZVerifying data now
126302024-02-20T22:46:39.094ZNo change after verify
126312024-02-20T22:46:39.094ZLoop: 58 Downstairs dump after verify (and repair):
126322024-02-20T22:46:39.106ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126332024-02-20T22:46:39.106Z 0 000-019 117 117 117 203 203 203 F F F
126342024-02-20T22:46:39.106Z 1 020-039 114 114 114 201 201 201 F F F
126352024-02-20T22:46:39.106Z 2 040-059 117 117 117 203 203 203 F F F
126362024-02-20T22:46:39.106Z 3 060-079 117 117 117 203 203 203 F F F
126372024-02-20T22:46:39.106Z 4 080-099 114 114 114 201 201 201 F F F
126382024-02-20T22:46:39.106Z 5 100-119 114 114 114 201 201 201 F F F
126392024-02-20T22:46:39.106Z 6 120-139 111 111 111 194 194 194 F F F
126402024-02-20T22:46:39.107Z 7 140-159 113 113 113 200 200 200 F F F
126412024-02-20T22:46:39.107Z 8 160-179 113 113 113 200 200 200 F F F
126422024-02-20T22:46:39.107Z 9 180-199 112 112 112 198 198 198 F F F
126432024-02-20T22:46:39.107Z 10 200-219 106 106 106 182 182 182 F F F
126442024-02-20T22:46:39.107Z 11 220-239 110 110 110 192 192 192 F F F
126452024-02-20T22:46:39.107Z 12 240-259 117 117 117 203 203 203 F F F
126462024-02-20T22:46:39.107Z 13 260-279 107 107 107 186 186 186 F F F
126472024-02-20T22:46:39.112Z 14 280-299 112 112 112 199 199 199 F F F
126482024-02-20T22:46:39.112Z 15 300-319 114 114 114 201 201 201 F F F
126492024-02-20T22:46:39.112Z 16 320-339 112 112 112 199 199 199 F F F
126502024-02-20T22:46:39.112Z 17 340-359 117 117 117 203 203 203 F F F
126512024-02-20T22:46:39.112Z 18 360-379 114 114 114 201 201 201 F F F
126522024-02-20T22:46:39.112Z 19 380-399 117 117 117 203 203 203 F F F
126532024-02-20T22:46:39.112Z 20 400-419 117 117 117 203 203 203 F F F
126542024-02-20T22:46:39.112Z 21 420-439 112 112 112 197 197 197 F F F
126552024-02-20T22:46:39.112Z 22 440-459 114 114 114 201 201 201 F F F
126562024-02-20T22:46:39.112Z 23 460-479 114 114 114 201 201 201 F F F
126572024-02-20T22:46:39.112Z 24 480-499 111 111 111 194 194 194 F F F
126582024-02-20T22:46:39.112Z 25 500-519 117 117 117 203 203 203 F F F
126592024-02-20T22:46:39.113Z 26 520-539 117 117 117 203 203 203 F F F
126602024-02-20T22:46:39.113Z 27 540-559 117 117 117 203 203 203 F F F
126612024-02-20T22:46:39.113Z 28 560-579 112 112 112 197 197 197 F F F
126622024-02-20T22:46:39.113Z 29 580-599 114 114 114 201 201 201 F F F
126632024-02-20T22:46:39.113ZMax gen: 117, Max flush: 203
126642024-02-20T22:46:39.113Z
126652024-02-20T22:46:39.113ZBegin loop 59
126662024-02-20T22:46:39.113ZDownstairs to restart: 0
126672024-02-20T22:46:39.131Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649129232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610}
126682024-02-20T22:46:39.131Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649433561Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610}
126692024-02-20T22:46:39.139Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.64946938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610}
126702024-02-20T22:46:39.139Z{"msg":"Crucible 33e826bc-f58a-4d9a-8114-94f82b3b024b has session id: d95eee81-ba38-4a1d-baeb-1142d114122e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649503889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126712024-02-20T22:46:39.139Z{"msg":"Upstairs opts: Upstairs UUID: 33e826bc-f58a-4d9a-8114-94f82b3b024b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649541997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126722024-02-20T22:46:39.139Z{"msg":"Crucible stats registered with UUID: 33e826bc-f58a-4d9a-8114-94f82b3b024b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649569267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"Crucible runtime is spawned
126732024-02-20T22:46:39.139Zsession_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126742024-02-20T22:46:39.139Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649700452Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610}
126752024-02-20T22:46:39.139Z{"msg":"33e826bc-f58a-4d9a-8114-94f82b3b024b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649841967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126762024-02-20T22:46:39.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649872745Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126772024-02-20T22:46:39.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649927544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126782024-02-20T22:46:39.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649958553Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126792024-02-20T22:46:39.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.649978102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126802024-02-20T22:46:39.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650066569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126812024-02-20T22:46:39.140Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650115397Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126822024-02-20T22:46:39.140Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650142496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126832024-02-20T22:46:39.140Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650200034Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126842024-02-20T22:46:39.140Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650248122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126852024-02-20T22:46:39.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650288381Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126862024-02-20T22:46:39.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650329979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126872024-02-20T22:46:39.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650359718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126882024-02-20T22:46:39.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650846451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126892024-02-20T22:46:39.140Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650908088Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126902024-02-20T22:46:39.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650957407Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126912024-02-20T22:46:39.140Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.650984636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126922024-02-20T22:46:39.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.651049963Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126932024-02-20T22:46:39.140Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.651081642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126942024-02-20T22:46:39.140Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.651552036Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126952024-02-20T22:46:39.141Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.651665012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126962024-02-20T22:46:39.141Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.65171525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126972024-02-20T22:46:39.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652170913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126982024-02-20T22:46:39.141Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652213432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
126992024-02-20T22:46:39.141Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.65225396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127002024-02-20T22:46:39.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652303249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127012024-02-20T22:46:39.141Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652356117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127022024-02-20T22:46:39.141Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652392006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127032024-02-20T22:46:39.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652425304Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127042024-02-20T22:46:39.141Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652460323Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127052024-02-20T22:46:39.141Z{"msg":"[0]R flush_numbers[0..12]: [203, 201, 203, 203, 201, 201, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652496102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127062024-02-20T22:46:39.141Z{"msg":"[0]R generation[0..12]: [117, 114, 117, 117, 114, 114, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652565509Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127072024-02-20T22:46:39.141Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO
127082024-02-20T22:46:39.141Zname":"crucible","level":30,"time":"2024-02-20T22:46:38.652611478Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127092024-02-20T22:46:39.141Z{"msg":"[1]R flush_numbers[0..12]: [203, 201, 203, 203, 201, 201, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652641466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127102024-02-20T22:46:39.142Z{"msg":"[1]R generation[0..12]: [117, 114, 117, 117, 114, 114, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652679885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127112024-02-20T22:46:39.142Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652744183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127122024-02-20T22:46:39.142Z{"msg":"[2]R flush_numbers[0..12]: [203, 201, 203, 203, 201, 201, 194, 200, 200, 198, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652786431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127132024-02-20T22:46:39.142Z{"msg":"[2]R generation[0..12]: [117, 114, 117, 117, 114, 114, 111, 113, 113, 112, 106, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.65282618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127142024-02-20T22:46:39.142Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652867358Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127152024-02-20T22:46:39.142Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652905767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127162024-02-20T22:46:39.142Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652944406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127172024-02-20T22:46:39.142Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.652982234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127182024-02-20T22:46:39.142Zdownstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127192024-02-20T22:46:39.142Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653013743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127202024-02-20T22:46:39.142Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653040122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127212024-02-20T22:46:39.142Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653053252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127222024-02-20T22:46:39.142Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653066831Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127232024-02-20T22:46:39.142Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.65308853Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127242024-02-20T22:46:39.143Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.6531028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127252024-02-20T22:46:39.143Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.65311654Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127262024-02-20T22:46:39.143Z{"msg":"33e826bc-f58a-4d9a-8114-94f82b3b024b is now active with session: d95eee81-ba38-4a1d-baeb-1142d114122e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653137169Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127272024-02-20T22:46:39.150Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653176207Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610,"session_id":"d95eee81-ba38-4a1d-baeb-1142d114122e"}
127282024-02-20T22:46:39.150Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:38.653198917Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610}
127292024-02-20T22:46:39.150ZLoading write count information from file "/tmp/test_repair_verify.data"
127302024-02-20T22:46:39.150ZRun Repair workload
127312024-02-20T22:46:39.150Z01/30 Write block 27 len 36864 data: 14 15 18 17 18 18 18 15 15
127322024-02-20T22:46:39.150Z02/30 Read block 280 len 36864
127332024-02-20T22:46:39.150Z03/30 Write block 49 len 36864 data: 9 7 7 9 7 10 8 7 6
127342024-02-20T22:46:39.150Z04/30 Write block 368 len 16384 data: 10 11 14 17
127352024-02-20T22:46:39.150Z05/30 Read block 281 len 28672
127362024-02-20T22:46:39.150Z06/30 Read block 310 len 28672
127372024-02-20T22:46:39.150Z07/30 Write block 384 len 4096 data: 7
127382024-02-20T22:46:39.150Z08/30 Read block 409 len 24576
127392024-02-20T22:46:39.150Z09/30 Write block 451 len 40960 data: 8 11 9 11 11 15 16 17 15 13
127402024-02-20T22:46:39.150Z10/30 Read block 548 len 32768
127412024-02-20T22:46:39.151Z11/30 Read block 254 len 8192
127422024-02-20T22:46:39.151Z12/30 Write block 229 len 36864 data: 12 16 14 14 16 16 13 13 12
127432024-02-20T22:46:39.155Z13/30 Read block 555 len 32768
127442024-02-20T22:46:39.155Z14/30 Flush
127452024-02-20T22:46:39.155Z15/30 Write block 203 len 16384 data: 7 8 9 10
127462024-02-20T22:46:39.155Z16/30 Write block 35 len 32768 data: 16 13 12 12 11 9 7 8
127472024-02-20T22:46:39.160Z17/30 Write block 259 len 40960 data: 7 7 7 7 6 10 10 11 11 10
127482024-02-20T22:46:39.160Z18/30 Read block 365 len 20480
127492024-02-20T22:46:39.160Z19/30 Write block 121 len 32768 data: 11 12 10 10 7 6 10 10
127502024-02-20T22:46:39.160Z20/30 Write block 578 len 24576 data: 13 15 16 13 13 11
127512024-02-20T22:46:39.160Z21/30 Read block 39 len 28672
127522024-02-20T22:46:39.164Z22/30 Flush
127532024-02-20T22:46:39.164Z23/30 Write block 231 len 32768 data: 15 15 17 17 14 14 13 8
127542024-02-20T22:46:39.164Z24/30 Read block 344 len 20480
127552024-02-20T22:46:39.174Z25/30 Read block 27 len 12288
127562024-02-20T22:46:39.174Z26/30 Write block 185 len 32768 data: 9 9 10 9 9 8 8 6
127572024-02-20T22:46:39.174Z27/30 Read block 23 len 12288
127582024-02-20T22:46:39.174Z28/30 Write block 94 len 24576 data: 8 7 5 7 4 9
127592024-02-20T22:46:39.174Z29/30 Write block 151 len 40960 data: 15 13 8 8 7 8 10 9 10 10
127602024-02-20T22:46:39.178Z30/30 Read block 270 len 40960
127612024-02-20T22:46:39.178Z----------------------------------------------------------------
127622024-02-20T22:46:39.178Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
127632024-02-20T22:46:39.178ZStates: Active Active Active
127642024-02-20T22:46:39.178ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127652024-02-20T22:46:39.178Z 1 Acked 1000 Write 9 Done Done Done false
127662024-02-20T22:46:39.178Z 2 Acked 1001 Read 9 Done Done Done false
127672024-02-20T22:46:39.178Z 3 Acked 1002 Write 9 Done Done Done false
127682024-02-20T22:46:39.178Z 4 Acked 1003 Write 4 Done Done Done false
127692024-02-20T22:46:39.178Z 5 Acked 1004 Read 7 Done Done Done false
127702024-02-20T22:46:39.179Z 6 Acked 1005 Read 7 Done Done Done false
127712024-02-20T22:46:39.179Z 7 Acked 1006 Write 1 Done Done Done false
127722024-02-20T22:46:39.179Z 8 Acked 1007 Read 6 Done Done Done false
127732024-02-20T22:46:39.179Z 9 Acked 1008 Write 10 Done Done Done false
127742024-02-20T22:46:39.179Z 10 Acked 1009 Read 8 Done Done Done false
127752024-02-20T22:46:39.179Z 11 Acked 1010 Read 2 Done Done Done false
127762024-02-20T22:46:39.179Z 12 Acked 1011 Write 9 Sent Done Done false
127772024-02-20T22:46:39.179Z 13 Acked 1012 Read 8 Sent Done Done false
127782024-02-20T22:46:39.179Z 14 Acked 1013 Flush 0 Sent Done Done false
127792024-02-20T22:46:39.179Z 15 Acked 1014 Write 4 Sent Done Done false
127802024-02-20T22:46:39.179Z 16 Acked 1015 Write 8 Sent Done Done false
127812024-02-20T22:46:39.179Z 17 Acked 1016 Write 10 Sent Done Done false
127822024-02-20T22:46:39.179Z 18 Acked 1017 Read 5 Sent Done Done false
127832024-02-20T22:46:39.179Z 19 Acked 1018 Write 8 Sent Done Done false
127842024-02-20T22:46:39.179Z 20 Acked 1019 Write 6 Sent Done Done false
127852024-02-20T22:46:39.179Z 21 Acked 1020 Read 7 Sent Done Done false
127862024-02-20T22:46:39.179Z 22 Acked 1021 Flush 0 Sent Done Done false
127872024-02-20T22:46:39.179Z 23 Acked 1022 Write 8 Sent Done Done false
127882024-02-20T22:46:39.179Z 24 Acked 1023 Read 5 Sent Done Done false
127892024-02-20T22:46:39.179Z 25 Acked 1024 Read 3 Sent Done Done false
127902024-02-20T22:46:39.179Z 26 Acked 1025 Write 8 Sent Done Done false
127912024-02-20T22:46:39.179Z 27 Acked 1026 Read 3 Sent Done Done false
127922024-02-20T22:46:39.179Z 28 Acked 1027 Write 6 Sent Done Done false
127932024-02-20T22:46:39.179Z 29 Acked 1028 Write 10 Sent Done Done false
127942024-02-20T22:46:39.179Z 30 Acked 1029 Read 10 Sent Sent Done false
127952024-02-20T22:46:39.179Z STATES DS:0 DS:1 DS:2 TOTAL
127962024-02-20T22:46:39.180Z New 0 0 0 0
127972024-02-20T22:46:39.180Z Sent 19 1 0 20
127982024-02-20T22:46:39.180Z Done 11 29 30 70
127992024-02-20T22:46:39.180Z Skipped 0 0 0 0
128002024-02-20T22:46:39.180Z Error 0 0 0 0
128012024-02-20T22:46:39.180ZLast Flush: 0 1021 1021
128022024-02-20T22:46:39.180ZDownstairs last five completed:
128032024-02-20T22:46:39.180ZUpstairs last five completed: 30 29 28 27 26
128042024-02-20T22:46:39.180Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:38.689665972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1610}
128052024-02-20T22:46:39.180ZWrote out file "/tmp/test_repair_verify.data"
128062024-02-20T22:46:39.180Z
128072024-02-20T22:46:39.184ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128082024-02-20T22:46:39.196ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128092024-02-20T22:46:39.196Z 0 000-019 117 117 117 203 203 203 F F F
128102024-02-20T22:46:39.196Z 1 020-039 114 118 118 201 205 205 T F F <---
128112024-02-20T22:46:39.196Z 2 040-059 117 118 118 203 205 205 T F F <---
128122024-02-20T22:46:39.197Z 3 060-079 117 117 117 203 203 203 F F F
128132024-02-20T22:46:39.197Z 4 080-099 114 114 114 201 201 201 F T T <---
128142024-02-20T22:46:39.197Z 5 100-119 114 114 114 201 201 201 F F F
128152024-02-20T22:46:39.197Z 6 120-139 111 118 118 194 205 205 F F F <---
128162024-02-20T22:46:39.197Z 7 140-159 113 113 113 200 200 200 F T T <---
128172024-02-20T22:46:39.197Z 8 160-179 113 113 113 200 200 200 F T T <---
128182024-02-20T22:46:39.205Z 9 180-199 112 112 112 198 198 198 F T T <---
128192024-02-20T22:46:39.205Z 10 200-219 106 118 118 182 205 205 F F F <---
128202024-02-20T22:46:39.205Z 11 220-239 110 118 118 192 204 204 F T T <---
128212024-02-20T22:46:39.205Z 12 240-259 117 118 118 203 205 205 F F F <---
128222024-02-20T22:46:39.205Z 13 260-279 107 118 118 186 205 205 F F F <---
128232024-02-20T22:46:39.205Z 14 280-299 112 112 112 199 199 199 F F F
128242024-02-20T22:46:39.205Z 15 300-319 114 114 114 201 201 201 F F F
128252024-02-20T22:46:39.205Z 16 320-339 112 112 112 199 199 199 F F F
128262024-02-20T22:46:39.205Z 17 340-359 117 117 117 203 203 203 F F F
128272024-02-20T22:46:39.205Z 18 360-379 114 118 118 201 204 204 T F F <---
128282024-02-20T22:46:39.205Z 19 380-399 117 118 118 203 204 204 T F F <---
128292024-02-20T22:46:39.205Z 20 400-419 117 117 117 203 203 203 F F F
128302024-02-20T22:46:39.205Z 21 420-439 112 112 112 197 197 197 F F F
128312024-02-20T22:46:39.205Z 22 440-459 114 118 118 201 204 204 T F F <---
128322024-02-20T22:46:39.205Z 23 460-479 114 118 118 201 204 204 T F F <---
128332024-02-20T22:46:39.205Z 24 480-499 111 111 111 194 194 194 F F F
128342024-02-20T22:46:39.205Z 25 500-519 117 117 117 203 203 203 F F F
128352024-02-20T22:46:39.206Z 26 520-539 117 117 117 203 203 203 F F F
128362024-02-20T22:46:39.206Z 27 540-559 117 117 117 203 203 203 F F F
128372024-02-20T22:46:39.206Z 28 560-579 112 118 118 197 205 205 F F F <---
128382024-02-20T22:46:39.206Z 29 580-599 114 118 118 201 205 205 F F F <---
128392024-02-20T22:46:39.206ZMax gen: 118, Max flush: 205
128402024-02-20T22:46:39.206ZError: Difference in extent metadata found!
128412024-02-20T22:46:39.206ZA Difference in extent metadata is expected here
128422024-02-20T22:46:39.206ZOn loop 59
128432024-02-20T22:46:39.206Z
128442024-02-20T22:46:39.206ZVerifying data now
128452024-02-20T22:46:45.021ZNo change after verify
128462024-02-20T22:46:45.021ZLoop: 59 Downstairs dump after verify (and repair):
128472024-02-20T22:46:45.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128482024-02-20T22:46:45.027Z 0 000-019 117 117 117 203 203 203 F F F
128492024-02-20T22:46:45.027Z 1 020-039 118 118 118 205 205 205 F F F
128502024-02-20T22:46:45.027Z 2 040-059 118 118 118 205 205 205 F F F
128512024-02-20T22:46:45.027Z 3 060-079 117 117 117 203 203 203 F F F
128522024-02-20T22:46:45.027Z 4 080-099 119 119 119 206 206 206 F F F
128532024-02-20T22:46:45.027Z 5 100-119 114 114 114 201 201 201 F F F
128542024-02-20T22:46:45.027Z 6 120-139 118 118 118 205 205 205 F F F
128552024-02-20T22:46:45.027Z 7 140-159 119 119 119 206 206 206 F F F
128562024-02-20T22:46:45.027Z 8 160-179 119 119 119 206 206 206 F F F
128572024-02-20T22:46:45.027Z 9 180-199 119 119 119 206 206 206 F F F
128582024-02-20T22:46:45.027Z 10 200-219 118 118 118 205 205 205 F F F
128592024-02-20T22:46:45.027Z 11 220-239 119 119 119 206 206 206 F F F
128602024-02-20T22:46:45.027Z 12 240-259 118 118 118 205 205 205 F F F
128612024-02-20T22:46:45.027Z 13 260-279 118 118 118 205 205 205 F F F
128622024-02-20T22:46:45.027Z 14 280-299 112 112 112 199 199 199 F F F
128632024-02-20T22:46:45.027Z 15 300-319 114 114 114 201 201 201 F F F
128642024-02-20T22:46:45.027Z 16 320-339 112 112 112 199 199 199 F F F
128652024-02-20T22:46:45.028Z 17 340-359 117 117 117 203 203 203 F F F
128662024-02-20T22:46:45.028Z 18 360-379 118 118 118 204 204 204 F F F
128672024-02-20T22:46:45.028Z 19 380-399 118 118 118 204 204 204 F F F
128682024-02-20T22:46:45.028Z 20 400-419 117 117 117 203 203 203 F F F
128692024-02-20T22:46:45.028Z 21 420-439 112 112 112 197 197 197 F F F
128702024-02-20T22:46:45.028Z 22 440-459 118 118 118 204 204 204 F F F
128712024-02-20T22:46:45.028Z 23 460-479 118 118 118 204 204 204 F F F
128722024-02-20T22:46:45.028Z 24 480-499 111 111 111 194 194 194 F F F
128732024-02-20T22:46:45.028Z 25 500-519 117 117 117 203 203 203 F F F
128742024-02-20T22:46:45.028Z 26 520-539 117 117 117 203 203 203 F F F
128752024-02-20T22:46:45.028Z 27 540-559 117 117 117 203 203 203 F F F
128762024-02-20T22:46:45.028Z 28 560-579 118 118 118 205 205 205 F F F
128772024-02-20T22:46:45.028Z 29 580-599 118 118 118 205 205 205 F F F
128782024-02-20T22:46:45.028ZMax gen: 119, Max flush: 206
128792024-02-20T22:46:45.032Z
128802024-02-20T22:46:45.033ZBegin loop 60
128812024-02-20T22:46:45.033ZDownstairs to restart: 0
128822024-02-20T22:46:45.047Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.564763823Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618}
128832024-02-20T22:46:45.053Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565084261Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618}
128842024-02-20T22:46:45.053Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565164159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618}
128852024-02-20T22:46:45.053Z{"msg":"Crucible ac4abbd9-19a8-4ae3-b46d-9fdfeebb96f1 has session id: c641816a-7aa5-4e0c-80e3-83e837bfd4ff","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565207337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128862024-02-20T22:46:45.053Z{"msg":"Upstairs opts: Upstairs UUID: ac4abbd9-19a8-4ae3-b46d-9fdfeebb96f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565255015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128872024-02-20T22:46:45.053Z{"msg":"Crucible stats registered with UUID: ac4abbd9-19a8-4ae3-b46d-9fdfeebb96f1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565281665Z"Crucible runtime is spawned
128882024-02-20T22:46:45.053Z,"hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128892024-02-20T22:46:45.053Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565386661Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618}
128902024-02-20T22:46:45.053Z{"msg":"ac4abbd9-19a8-4ae3-b46d-9fdfeebb96f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565472667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128912024-02-20T22:46:45.053Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565499647Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128922024-02-20T22:46:45.054Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565516156Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128932024-02-20T22:46:45.054Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565530225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128942024-02-20T22:46:45.054Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565544505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128952024-02-20T22:46:45.054Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565571044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128962024-02-20T22:46:45.054Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565599073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128972024-02-20T22:46:45.054Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565632752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128982024-02-20T22:46:45.054Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.56567892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
128992024-02-20T22:46:45.054Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565709799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129002024-02-20T22:46:45.054Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565750568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129012024-02-20T22:46:45.054Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565780207Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129022024-02-20T22:46:45.054Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.565915772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129032024-02-20T22:46:45.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.566437703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129042024-02-20T22:46:45.054Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.566477092Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129052024-02-20T22:46:45.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.566494581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129062024-02-20T22:46:45.054Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.56650971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129072024-02-20T22:46:45.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.566552939Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129082024-02-20T22:46:45.054Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.566602507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129092024-02-20T22:46:45.054Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.5670909Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129102024-02-20T22:46:45.054Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567130968Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129112024-02-20T22:46:45.054Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567397689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129122024-02-20T22:46:45.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567588112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129132024-02-20T22:46:45.054Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.56764069Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129142024-02-20T22:46:45.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567696708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129152024-02-20T22:46:45.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567729697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129162024-02-20T22:46:45.055Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567780445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129172024-02-20T22:46:45.055Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567832663Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129182024-02-20T22:46:45.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567861862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129192024-02-20T22:46:45.055Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567896361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129202024-02-20T22:46:45.055Z{"msg":"[0]R flush_numbers[0..12]: [203, 205, 205, 203, 206, 201, 205, 206, 206, 206, 205, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.56792159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129212024-02-20T22:46:45.055Z{"msg":"[0]R generation[0..12]: [117, 118, 118, 117, 119, 114, 118, 119, 119, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567958019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129222024-02-20T22:46:45.055Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.567994257Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129232024-02-20T22:46:45.055Z{"msg":"[1]R flush_numbers[0..12]: [203, 205, 205, 203, 206, 201, 205, 206, 206, 206, 205, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568031906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129242024-02-20T22:46:45.055Z{"msg":"[1]R generation[0..12]: [117, 118, 118, 117, 119, 114, 118, 119, 119, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568067485Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129252024-02-20T22:46:45.055ZWait for a query_work_queue command to finish before sending IO
129262024-02-20T22:46:45.055Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568111673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129272024-02-20T22:46:45.055Z{"msg":"[2]R flush_numbers[0..12]: [203, 205, 205, 203, 206, 201, 205, 206, 206, 206, 205, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568154622Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129282024-02-20T22:46:45.055Z{"msg":"[2]R generation[0..12]: [117, 118, 118, 117, 119, 114, 118, 119, 119, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.56819275Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129292024-02-20T22:46:45.055Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568230329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129302024-02-20T22:46:45.055Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568267758Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129312024-02-20T22:46:45.055Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568307796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129322024-02-20T22:46:45.055Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568343605Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129332024-02-20T22:46:45.055Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568378524Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129342024-02-20T22:46:45.055Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129352024-02-20T22:46:45.055Z,"time":"2024-02-20T22:46:44.568415372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129362024-02-20T22:46:45.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568451831Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129372024-02-20T22:46:45.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.56847792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129382024-02-20T22:46:45.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568492599Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129392024-02-20T22:46:45.055Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568513179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129402024-02-20T22:46:45.056Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568525968Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129412024-02-20T22:46:45.056Z{"msg":"ac4abbd9-19a8-4ae3-b46d-9fdfeebb96f1 is now active with session: c641816a-7aa5-4e0c-80e3-83e837bfd4ff","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568545328Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129422024-02-20T22:46:45.056Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568558147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618,"session_id":"c641816a-7aa5-4e0c-80e3-83e837bfd4ff"}
129432024-02-20T22:46:45.056Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:44.568577036Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618}
129442024-02-20T22:46:45.056ZLoading write count information from file "/tmp/test_repair_verify.data"
129452024-02-20T22:46:45.056ZRun Repair workload
129462024-02-20T22:46:45.060Z01/30 Write block 570 len 8192 data: 6 4
129472024-02-20T22:46:45.060Z02/30 Write block 80 len 4096 data: 7
129482024-02-20T22:46:45.060Z03/30 Read block 207 len 12288
129492024-02-20T22:46:45.060Z04/30 Read block 489 len 32768
129502024-02-20T22:46:45.060Z05/30 Write block 463 len 40960 data: 15 12 12 10 10 12 14 10 9 10
129512024-02-20T22:46:45.060Z06/30 Read block 409 len 16384
129522024-02-20T22:46:45.060Z07/30 Write block 251 len 36864 data: 6 6 9 8 6 5 6 8 8
129532024-02-20T22:46:45.060Z08/30 Read block 443 len 8192
129542024-02-20T22:46:45.060Z09/30 Read block 335 len 32768
129552024-02-20T22:46:45.060Z10/30 Read block 222 len 36864
129562024-02-20T22:46:45.064Z11/30 Read block 154 len 40960
129572024-02-20T22:46:45.064Z12/30 Flush
129582024-02-20T22:46:45.065Z13/30 Write block 461 len 20480 data: 15 15 16 13 13
129592024-02-20T22:46:45.065Z14/30 Flush
129602024-02-20T22:46:45.068Z15/30 Write block 232 len 24576 data: 16 18 18 15 15 14
129612024-02-20T22:46:45.072Z16/30 Read block 539 len 24576
129622024-02-20T22:46:45.072Z17/30 Read block 487 len 16384
129632024-02-20T22:46:45.072Z18/30 Write block 397 len 28672 data: 15 9 11 11 9 11 12
129642024-02-20T22:46:45.072Z19/30 Write block 222 len 36864 data: 7 7 6 5 6 7 9 13 17
129652024-02-20T22:46:45.072Z20/30 Read block 340 len 32768
129662024-02-20T22:46:45.076Z21/30 Write block 476 len 36864 data: 11 15 16 12 15 14 14 14 12
129672024-02-20T22:46:45.076Z22/30 Write block 111 len 4096 data: 11
129682024-02-20T22:46:45.076Z23/30 Read block 127 len 40960
129692024-02-20T22:46:45.076Z24/30 Write block 379 len 24576 data: 8 8 7 9 7 8
129702024-02-20T22:46:45.080Z25/30 Write block 114 len 24576 data: 9 9 10 11 6 10
129712024-02-20T22:46:45.080Z26/30 Write block 75 len 24576 data: 7 6 7 8 7 8
129722024-02-20T22:46:45.080Z27/30 Write block 15 len 8192 data: 7 13
129732024-02-20T22:46:45.081Z28/30 Read block 565 len 16384
129742024-02-20T22:46:45.081Z29/30 Write block 275 len 12288 data: 5 4 5
129752024-02-20T22:46:45.081Z30/30 Read block 400 len 8192
129762024-02-20T22:46:45.085Z----------------------------------------------------------------
129772024-02-20T22:46:45.085Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129782024-02-20T22:46:45.085ZStates: Active Active Active
129792024-02-20T22:46:45.085ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129802024-02-20T22:46:45.085Z 1 Acked 1000 Write 2 Done Done Done false
129812024-02-20T22:46:45.085Z 2 Acked 1001 Write 1 Done Done Done false
129822024-02-20T22:46:45.085Z 3 Acked 1002 Read 3 Done Done Done false
129832024-02-20T22:46:45.085Z 4 Acked 1003 Read 8 Done Done Done false
129842024-02-20T22:46:45.085Z 5 Acked 1004 Write 10 Done Done Done false
129852024-02-20T22:46:45.085Z 6 Acked 1005 Read 4 Done Done Done false
129862024-02-20T22:46:45.085Z 7 Acked 1006 Write 9 Sent Done Done false
129872024-02-20T22:46:45.085Z 8 Acked 1007 Read 2 Sent Done Done false
129882024-02-20T22:46:45.085Z 9 Acked 1008 Read 8 Sent Done Done false
129892024-02-20T22:46:45.086Z 10 Acked 1009 Read 9 Sent Done Done false
129902024-02-20T22:46:45.086Z 11 Acked 1010 Read 10 Sent Done Done false
129912024-02-20T22:46:45.086Z 12 Acked 1011 Flush 0 Sent Done Done false
129922024-02-20T22:46:45.086Z 13 Acked 1012 Write 5 Sent Done Done false
129932024-02-20T22:46:45.086Z 14 Acked 1013 Flush 0 Sent Done Done false
129942024-02-20T22:46:45.086Z 15 Acked 1014 Write 6 Sent Done Done false
129952024-02-20T22:46:45.086Z 16 Acked 1015 Read 6 Sent Done Done false
129962024-02-20T22:46:45.086Z 17 Acked 1016 Read 4 Sent Done Done false
129972024-02-20T22:46:45.086Z 18 Acked 1017 Write 7 Sent Done Done false
129982024-02-20T22:46:45.086Z 19 Acked 1018 Write 9 Sent Done Done false
129992024-02-20T22:46:45.086Z 20 Acked 1019 Read 8 Sent Done Done false
130002024-02-20T22:46:45.086Z 21 Acked 1020 Write 9 Sent Done Done false
130012024-02-20T22:46:45.086Z 22 Acked 1021 Write 1 Sent Done Done false
130022024-02-20T22:46:45.086Z 23 Acked 1022 Read 10 Sent Done Done false
130032024-02-20T22:46:45.086Z 24 Acked 1023 Write 6 Sent Done Done false
130042024-02-20T22:46:45.086Z 25 Acked 1024 Write 6 Sent Done Done false
130052024-02-20T22:46:45.086Z 26 Acked 1025 Write 6 Sent Done Done false
130062024-02-20T22:46:45.086Z 27 Acked 1026 Write 2 Sent Done Done false
130072024-02-20T22:46:45.086Z 28 Acked 1027 Read 4 Sent Done Done false
130082024-02-20T22:46:45.086Z 29 Acked 1028 Write 3 Sent Done Done false
130092024-02-20T22:46:45.086Z 30 Acked 1029 Read 2 Sent Done Done false
130102024-02-20T22:46:45.086Z STATES DS:0 DS:1 DS:2 TOTAL
130112024-02-20T22:46:45.086Z New 0 0 0 0
130122024-02-20T22:46:45.086Z Sent 24 0 0 24
130132024-02-20T22:46:45.086Z Done 6 30 30 66
130142024-02-20T22:46:45.087Z Skipped 0 0 0 0
130152024-02-20T22:46:45.087Z Error 0 0 0 0
130162024-02-20T22:46:45.087ZLast Flush: 0 1013 1013
130172024-02-20T22:46:45.087ZDownstairs last five completed:
130182024-02-20T22:46:45.087ZUpstairs last five completed: 30 29 28 27 26
130192024-02-20T22:46:45.087Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:44.598997058Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1618}
130202024-02-20T22:46:45.087ZWrote out file "/tmp/test_repair_verify.data"
130212024-02-20T22:46:45.090Z
130222024-02-20T22:46:45.090ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130232024-02-20T22:46:45.114ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130242024-02-20T22:46:45.114Z 0 000-019 117 117 117 203 203 203 F T T <---
130252024-02-20T22:46:45.114Z 1 020-039 118 118 118 205 205 205 F F F
130262024-02-20T22:46:45.114Z 2 040-059 118 118 118 205 205 205 F F F
130272024-02-20T22:46:45.114Z 3 060-079 117 117 117 203 203 203 F T T <---
130282024-02-20T22:46:45.115Z 4 080-099 119 120 120 206 207 207 T T T <---
130292024-02-20T22:46:45.115Z 5 100-119 114 114 114 201 201 201 F T T <---
130302024-02-20T22:46:45.115Z 6 120-139 118 118 118 205 205 205 F F F
130312024-02-20T22:46:45.115Z 7 140-159 119 119 119 206 206 206 F F F
130322024-02-20T22:46:45.116Z 8 160-179 119 119 119 206 206 206 F F F
130332024-02-20T22:46:45.116Z 9 180-199 119 119 119 206 206 206 F F F
130342024-02-20T22:46:45.116Z 10 200-219 118 118 118 205 205 205 F F F
130352024-02-20T22:46:45.539Z 11 220-239 119 119 119 206 206 206 F T T <---
130362024-02-20T22:46:45.539Z 12 240-259 118 120 120 205 207 207 F F F <---
130372024-02-20T22:46:45.539Z 13 260-279 118 118 118 205 205 205 F T T <---
130382024-02-20T22:46:45.539Z 14 280-299 112 112 112 199 199 199 F F F
130392024-02-20T22:46:45.539Z 15 300-319 114 114 114 201 201 201 F F F
130402024-02-20T22:46:45.539Z 16 320-339 112 112 112 199 199 199 F F F
130412024-02-20T22:46:45.539Z 17 340-359 117 117 117 203 203 203 F F F
130422024-02-20T22:46:45.539Z 18 360-379 118 118 118 204 204 204 F T T <---
130432024-02-20T22:46:45.539Z 19 380-399 118 118 118 204 204 204 F T T <---
130442024-02-20T22:46:45.539Z 20 400-419 117 117 117 203 203 203 F T T <---
130452024-02-20T22:46:45.540Z 21 420-439 112 112 112 197 197 197 F F F
130462024-02-20T22:46:45.540Z 22 440-459 118 118 118 204 204 204 F F F
130472024-02-20T22:46:45.540Z 23 460-479 118 120 120 204 208 208 T T T <---
130482024-02-20T22:46:45.540Z 24 480-499 111 111 111 194 194 194 F T T <---
130492024-02-20T22:46:45.540Z 25 500-519 117 117 117 203 203 203 F F F
130502024-02-20T22:46:45.540Z 26 520-539 117 117 117 203 203 203 F F F
130512024-02-20T22:46:45.540Z 27 540-559 117 117 117 203 203 203 F F F
130522024-02-20T22:46:45.540Z 28 560-579 118 120 120 205 207 207 T F F <---
130532024-02-20T22:46:45.540Z 29 580-599 118 118 118 205 205 205 F F F
130542024-02-20T22:46:45.540ZMax gen: 120, Max flush: 208
130552024-02-20T22:46:45.540ZError: Difference in extent metadata found!
130562024-02-20T22:46:45.540ZA Difference in extent metadata is expected here
130572024-02-20T22:46:45.540ZOn loop 60
130582024-02-20T22:46:45.540Z
130592024-02-20T22:46:45.541ZVerifying data now
130602024-02-20T22:46:49.774ZNo change after verify
130612024-02-20T22:46:49.774ZLoop: 60 Downstairs dump after verify (and repair):
130622024-02-20T22:46:49.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130632024-02-20T22:46:49.789Z 0 000-019 121 121 121 209 209 209 F F F
130642024-02-20T22:46:49.789Z 1 020-039 118 118 118 205 205 205 F F F
130652024-02-20T22:46:49.789Z 2 040-059 118 118 118 205 205 205 F F F
130662024-02-20T22:46:49.789Z 3 060-079 121 121 121 209 209 209 F F F
130672024-02-20T22:46:49.789Z 4 080-099 121 121 121 209 209 209 F F F
130682024-02-20T22:46:49.789Z 5 100-119 121 121 121 209 209 209 F F F
130692024-02-20T22:46:49.789Z 6 120-139 118 118 118 205 205 205 F F F
130702024-02-20T22:46:49.789Z 7 140-159 119 119 119 206 206 206 F F F
130712024-02-20T22:46:49.789Z 8 160-179 119 119 119 206 206 206 F F F
130722024-02-20T22:46:49.789Z 9 180-199 119 119 119 206 206 206 F F F
130732024-02-20T22:46:49.789Z 10 200-219 118 118 118 205 205 205 F F F
130742024-02-20T22:46:49.789Z 11 220-239 121 121 121 209 209 209 F F F
130752024-02-20T22:46:49.793Z 12 240-259 120 120 120 207 207 207 F F F
130762024-02-20T22:46:49.793Z 13 260-279 121 121 121 209 209 209 F F F
130772024-02-20T22:46:49.793Z 14 280-299 112 112 112 199 199 199 F F F
130782024-02-20T22:46:49.794Z 15 300-319 114 114 114 201 201 201 F F F
130792024-02-20T22:46:49.794Z 16 320-339 112 112 112 199 199 199 F F F
130802024-02-20T22:46:49.794Z 17 340-359 117 117 117 203 203 203 F F F
130812024-02-20T22:46:49.794Z 18 360-379 121 121 121 209 209 209 F F F
130822024-02-20T22:46:49.794Z 19 380-399 121 121 121 209 209 209 F F F
130832024-02-20T22:46:49.794Z 20 400-419 121 121 121 209 209 209 F F F
130842024-02-20T22:46:49.794Z 21 420-439 112 112 112 197 197 197 F F F
130852024-02-20T22:46:49.794Z 22 440-459 118 118 118 204 204 204 F F F
130862024-02-20T22:46:49.794Z 23 460-479 121 121 121 209 209 209 F F F
130872024-02-20T22:46:49.794Z 24 480-499 121 121 121 209 209 209 F F F
130882024-02-20T22:46:49.794Z 25 500-519 117 117 117 203 203 203 F F F
130892024-02-20T22:46:49.794Z 26 520-539 117 117 117 203 203 203 F F F
130902024-02-20T22:46:49.794Z 27 540-559 117 117 117 203 203 203 F F F
130912024-02-20T22:46:49.794Z 28 560-579 120 120 120 207 207 207 F F F
130922024-02-20T22:46:49.794Z 29 580-599 118 118 118 205 205 205 F F F
130932024-02-20T22:46:49.794ZMax gen: 121, Max flush: 209
130942024-02-20T22:46:49.794Z
130952024-02-20T22:46:49.794ZBegin loop 61
130962024-02-20T22:46:49.794ZDownstairs to restart: 0
130972024-02-20T22:46:49.810Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.32747266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626}
130982024-02-20T22:46:49.810Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.327769299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626}
130992024-02-20T22:46:49.811Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.327796618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626}
131002024-02-20T22:46:49.811Z{"Crucible runtime is spawned
131012024-02-20T22:46:49.817Zmsg":"Crucible 5148b1f2-0ef9-4a16-bcc5-8e6ee3c647a7 has session id: aa2836c3-6361-4e47-afdc-6770c0983ae4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.327843387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131022024-02-20T22:46:49.817Z{"msg":"Upstairs opts: Upstairs UUID: 5148b1f2-0ef9-4a16-bcc5-8e6ee3c647a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.327892565Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131032024-02-20T22:46:49.818Z{"msg":"Crucible stats registered with UUID: 5148b1f2-0ef9-4a16-bcc5-8e6ee3c647a7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.327918504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131042024-02-20T22:46:49.818Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.327965652Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626}
131052024-02-20T22:46:49.818Z{"msg":"5148b1f2-0ef9-4a16-bcc5-8e6ee3c647a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.32802953Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131062024-02-20T22:46:49.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328061189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131072024-02-20T22:46:49.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328078728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131082024-02-20T22:46:49.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328108977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131092024-02-20T22:46:49.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328124317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131102024-02-20T22:46:49.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328145486Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131112024-02-20T22:46:49.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328159485Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131122024-02-20T22:46:49.818Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328180664Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131132024-02-20T22:46:49.818Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328211023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131142024-02-20T22:46:49.818Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328245732Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131152024-02-20T22:46:49.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328433335Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131162024-02-20T22:46:49.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328485633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131172024-02-20T22:46:49.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.328515472Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131182024-02-20T22:46:49.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329172699Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131192024-02-20T22:46:49.818Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329212007Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131202024-02-20T22:46:49.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329228937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131212024-02-20T22:46:49.818Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329244386Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131222024-02-20T22:46:49.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329277555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131232024-02-20T22:46:49.818Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329316524Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131242024-02-20T22:46:49.818Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329734049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131252024-02-20T22:46:49.818Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.329792697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131262024-02-20T22:46:49.818Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330116465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131272024-02-20T22:46:49.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330316838Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131282024-02-20T22:46:49.819Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330395145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131292024-02-20T22:46:49.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330450753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131302024-02-20T22:46:49.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330487232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131312024-02-20T22:46:49.819Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.3305242Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131322024-02-20T22:46:49.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330567759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131332024-02-20T22:46:49.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330615027Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131342024-02-20T22:46:49.819Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330665006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131352024-02-20T22:46:49.819Z{"msg":"[0]R flush_numbers[0..12]: [209, 205, 205, 209, 209, 209, 205, 206, 206, 206, 205, 209]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330704444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131362024-02-20T22:46:49.819Z{"msg":"[0]R generation[0..12]: [121, 118, 118, 121, 121, 121, 118, 119, 119, 119, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330723413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131372024-02-20T22:46:49.819Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330750083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131382024-02-20T22:46:49.819Z{"msg":"[1]R flush_numbers[0..12]: [209, 205, 205, 209, 209, 209, 205, 206, 206, 206, 205, 209]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330781391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131392024-02-20T22:46:49.819Z{"msg":"[1]R generation[0..12]: [121, 118, 118, 121, 121, 121, 118, 119, 119, 119, 118, 121]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
131402024-02-20T22:46:49.819Z,"time":"2024-02-20T22:46:49.33081202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131412024-02-20T22:46:49.819Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330849649Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131422024-02-20T22:46:49.819Z{"msg":"[2]R flush_numbers[0..12]: [209, 205, 205, 209, 209, 209, 205, 206, 206, 206, 205, 209]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330883178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131432024-02-20T22:46:49.819Z{"msg":"[2]R generation[0..12]: [121, 118, 118, 121, 121, 121, 118, 119, 119, 119, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330916277Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131442024-02-20T22:46:49.819Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330949675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131452024-02-20T22:46:49.819Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.330982074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131462024-02-20T22:46:49.819Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331026142Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131472024-02-20T22:46:49.819Z{"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331042632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131482024-02-20T22:46:49.819Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331056632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131492024-02-20T22:46:49.819Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331070971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131502024-02-20T22:46:49.819Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331083351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131512024-02-20T22:46:49.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.33110374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131522024-02-20T22:46:49.820Z{"msg":"Transition from WaitQuorum to Active","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131532024-02-20T22:46:49.820Z,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331120519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131542024-02-20T22:46:49.820Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331140969Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131552024-02-20T22:46:49.820Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331153688Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131562024-02-20T22:46:49.820Z{"msg":"5148b1f2-0ef9-4a16-bcc5-8e6ee3c647a7 is now active with session: aa2836c3-6361-4e47-afdc-6770c0983ae4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331173227Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131572024-02-20T22:46:49.820Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331185467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626,"session_id":"aa2836c3-6361-4e47-afdc-6770c0983ae4"}
131582024-02-20T22:46:49.820Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:49.331198076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626}
131592024-02-20T22:46:49.820ZLoading write count information from file "/tmp/test_repair_verify.data"
131602024-02-20T22:46:49.820ZRun Repair workload
131612024-02-20T22:46:49.820Z01/30 Write block 22 len 8192 data: 11 9
131622024-02-20T22:46:49.827Z02/30 Write block 120 len 28672 data: 11 12 13 11 11 8 7
131632024-02-20T22:46:49.827Z03/30 Read block 50 len 28672
131642024-02-20T22:46:49.827Z04/30 Read block 72 len 20480
131652024-02-20T22:46:49.827Z05/30 Read block 84 len 4096
131662024-02-20T22:46:49.827Z06/30 Read block 78 len 12288
131672024-02-20T22:46:49.827Z07/30 Read block 295 len 20480
131682024-02-20T22:46:49.827Z08/30 Write block 410 len 24576 data: 11 13 13 16 17 16
131692024-02-20T22:46:49.827Z09/30 Flush
131702024-02-20T22:46:49.827Z10/30 Write block 297 len 32768 data: 7 10 10 9 13 15 15 14
131712024-02-20T22:46:49.827Z11/30 Read block 246 len 16384
131722024-02-20T22:46:49.832Z12/30 Read block 146 len 32768
131732024-02-20T22:46:49.832Z13/30 Read block 172 len 32768
131742024-02-20T22:46:49.832Z14/30 Read block 336 len 40960
131752024-02-20T22:46:49.832Z15/30 Read block 207 len 8192
131762024-02-20T22:46:49.832Z16/30 Read block 456 len 24576
131772024-02-20T22:46:49.832Z17/30 Flush
131782024-02-20T22:46:49.837Z18/30 Write block 192 len 24576 data: 7 6 6 8 7 8
131792024-02-20T22:46:49.837Z19/30 Write block 576 len 32768 data: 8 11 14 16 17 14 14 12
131802024-02-20T22:46:49.837Z20/30 Write block 319 len 32768 data: 11 13 11 14 10 7 12 13
131812024-02-20T22:46:49.837Z21/30 Write block 459 len 28672 data: 16 14 16 16 17 14 14
131822024-02-20T22:46:49.837Z22/30 Write block 98 len 4096 data: 5
131832024-02-20T22:46:49.837Z23/30 Write block 49 len 16384 data: 10 8 8 10
131842024-02-20T22:46:49.837Z24/30 Write block 333 len 12288 data: 15 13 11
131852024-02-20T22:46:49.837Z25/30 Read block 322 len 16384
131862024-02-20T22:46:49.843Z26/30 Read block 348 len 24576
131872024-02-20T22:46:49.843Z27/30 Write block 65 len 36864 data: 8 7 11 10 11 13 13 11 9
131882024-02-20T22:46:49.843Z28/30 Write block 467 len 16384 data: 11 13 15 11
131892024-02-20T22:46:49.843Z29/30 Write block 573 len 8192 data: 7 7
131902024-02-20T22:46:49.843Z30/30 Write block 524 len 24576 data: 6 7 10 11 9 9
131912024-02-20T22:46:49.843Z----------------------------------------------------------------
131922024-02-20T22:46:49.843Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
131932024-02-20T22:46:49.843ZStates: Active Active Active
131942024-02-20T22:46:49.843ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131952024-02-20T22:46:49.843Z 1 Acked 1000 Write 2 Done Done Done false
131962024-02-20T22:46:49.843Z 2 Acked 1001 Write 7 Done Done Done false
131972024-02-20T22:46:49.843Z 3 Acked 1002 Read 7 Sent Done Done false
131982024-02-20T22:46:49.843Z 4 Acked 1003 Read 5 Sent Done Done false
131992024-02-20T22:46:49.843Z 5 Acked 1004 Read 1 Sent Done Done false
132002024-02-20T22:46:49.843Z 6 Acked 1005 Read 3 Sent Done Done false
132012024-02-20T22:46:49.843Z 7 Acked 1006 Read 5 Sent Done Done false
132022024-02-20T22:46:49.843Z 8 Acked 1007 Write 6 Sent Done Done false
132032024-02-20T22:46:49.843Z 9 Acked 1008 Flush 0 Sent Done Done false
132042024-02-20T22:46:49.843Z 10 Acked 1009 Write 8 Sent Done Done false
132052024-02-20T22:46:49.843Z 11 Acked 1010 Read 4 Sent Done Done false
132062024-02-20T22:46:49.843Z 12 Acked 1011 Read 8 Sent Done Done false
132072024-02-20T22:46:49.843Z 13 Acked 1012 Read 8 Sent Done Done false
132082024-02-20T22:46:49.843Z 14 Acked 1013 Read 10 Sent Done Done false
132092024-02-20T22:46:49.844Z 15 Acked 1014 Read 2 Sent Done Done false
132102024-02-20T22:46:49.844Z 16 Acked 1015 Read 6 Sent Done Done false
132112024-02-20T22:46:49.844Z 17 Acked 1016 Flush 0 Sent Done Done false
132122024-02-20T22:46:49.844Z 18 Acked 1017 Write 6 Sent Done Done false
132132024-02-20T22:46:49.844Z 19 Acked 1018 Write 8 Sent Done Done false
132142024-02-20T22:46:49.844Z 20 Acked 1019 Write 8 Sent Done Done false
132152024-02-20T22:46:49.844Z 21 Acked 1020 Write 7 Sent Done Done false
132162024-02-20T22:46:49.844Z 22 Acked 1021 Write 1 Sent Done Done false
132172024-02-20T22:46:49.844Z 23 Acked 1022 Write 4 Sent Done Done false
132182024-02-20T22:46:49.844Z 24 Acked 1023 Write 3 Sent Done Done false
132192024-02-20T22:46:49.844Z 25 Acked 1024 Read 4 Sent Done Done false
132202024-02-20T22:46:49.844Z 26 Acked 1025 Read 6 Sent Done Done false
132212024-02-20T22:46:49.844Z 27 Acked 1026 Write 9 Sent Done Done false
132222024-02-20T22:46:49.844Z 28 Acked 1027 Write 4 Sent Done Done false
132232024-02-20T22:46:49.844Z 29 Acked 1028 Write 2 Sent Done Done false
132242024-02-20T22:46:49.844Z 30 Acked 1029 Write 6 Sent Sent Sent false
132252024-02-20T22:46:49.844Z STATES DS:0 DS:1 DS:2 TOTAL
132262024-02-20T22:46:49.844Z New 0 0 0 0
132272024-02-20T22:46:49.844Z Sent 28 1 1 30
132282024-02-20T22:46:49.844Z Done 2 29 29 60
132292024-02-20T22:46:49.844Z Skipped 0 0 0 0
132302024-02-20T22:46:49.844Z Error 0 0 0 0
132312024-02-20T22:46:49.844ZLast Flush: 0 1016 1016
132322024-02-20T22:46:49.844ZDownstairs last five completed:
132332024-02-20T22:46:49.845ZUpstairs last five completed: 30 29 28 27 26
132342024-02-20T22:46:49.845Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:49.359186115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1626}
132352024-02-20T22:46:49.849ZWrote out file "/tmp/test_repair_verify.data"
132362024-02-20T22:46:49.849Z
132372024-02-20T22:46:49.853ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132382024-02-20T22:46:49.868ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132392024-02-20T22:46:49.868Z 0 000-019 121 121 121 209 209 209 F F F
132402024-02-20T22:46:49.868Z 1 020-039 118 122 122 205 210 210 T F F <---
132412024-02-20T22:46:49.868Z 2 040-059 118 118 118 205 205 205 F T T <---
132422024-02-20T22:46:49.869Z 3 060-079 121 121 121 209 209 209 F T T <---
132432024-02-20T22:46:49.869Z 4 080-099 121 121 121 209 209 209 F T T <---
132442024-02-20T22:46:49.869Z 5 100-119 121 121 121 209 209 209 F F F
132452024-02-20T22:46:49.869Z 6 120-139 118 122 122 205 210 210 T F F <---
132462024-02-20T22:46:49.869Z 7 140-159 119 119 119 206 206 206 F F F
132472024-02-20T22:46:49.873Z 8 160-179 119 119 119 206 206 206 F F F
132482024-02-20T22:46:49.873Z 9 180-199 119 119 119 206 206 206 F T T <---
132492024-02-20T22:46:49.873Z 10 200-219 118 118 118 205 205 205 F F F
132502024-02-20T22:46:49.873Z 11 220-239 121 121 121 209 209 209 F F F
132512024-02-20T22:46:49.873Z 12 240-259 120 120 120 207 207 207 F F F
132522024-02-20T22:46:49.873Z 13 260-279 121 121 121 209 209 209 F F F
132532024-02-20T22:46:49.873Z 14 280-299 112 122 122 199 211 211 F F F <---
132542024-02-20T22:46:49.873Z 15 300-319 114 122 122 201 211 211 F T T <---
132552024-02-20T22:46:49.873Z 16 320-339 112 112 112 199 199 199 F T T <---
132562024-02-20T22:46:49.873Z 17 340-359 117 117 117 203 203 203 F F F
132572024-02-20T22:46:49.874Z 18 360-379 121 121 121 209 209 209 F F F
132582024-02-20T22:46:49.874Z 19 380-399 121 121 121 209 209 209 F F F
132592024-02-20T22:46:49.874Z 20 400-419 121 122 122 209 210 210 F F F <---
132602024-02-20T22:46:49.874Z 21 420-439 112 112 112 197 197 197 F F F
132612024-02-20T22:46:49.874Z 22 440-459 118 118 118 204 204 204 F T T <---
132622024-02-20T22:46:49.874Z 23 460-479 121 121 121 209 209 209 F T T <---
132632024-02-20T22:46:49.874Z 24 480-499 121 121 121 209 209 209 F F F
132642024-02-20T22:46:49.874Z 25 500-519 117 117 117 203 203 203 F F F
132652024-02-20T22:46:49.874Z 26 520-539 117 117 117 203 203 203 F T T <---
132662024-02-20T22:46:49.874Z 27 540-559 117 117 117 203 203 203 F F F
132672024-02-20T22:46:49.874Z 28 560-579 120 120 120 207 207 207 F T T <---
132682024-02-20T22:46:49.874Z 29 580-599 118 118 118 205 205 205 F T T <---
132692024-02-20T22:46:49.874ZMax gen: 122, Max flush: 211
132702024-02-20T22:46:49.874ZError: Difference in extent metadata found!
132712024-02-20T22:46:49.874ZA Difference in extent metadata is expected here
132722024-02-20T22:46:49.874ZOn loop 61
132732024-02-20T22:46:49.874Z
132742024-02-20T22:46:49.878ZVerifying data now
132752024-02-20T22:46:55.017ZNo change after verify
132762024-02-20T22:46:55.017ZLoop: 61 Downstairs dump after verify (and repair):
132772024-02-20T22:46:55.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132782024-02-20T22:46:55.025Z 0 000-019 121 121 121 209 209 209 F F F
132792024-02-20T22:46:55.025Z 1 020-039 122 122 122 210 210 210 F F F
132802024-02-20T22:46:55.025Z 2 040-059 123 123 123 212 212 212 F F F
132812024-02-20T22:46:55.025Z 3 060-079 123 123 123 212 212 212 F F F
132822024-02-20T22:46:55.025Z 4 080-099 123 123 123 212 212 212 F F F
132832024-02-20T22:46:55.025Z 5 100-119 121 121 121 209 209 209 F F F
132842024-02-20T22:46:55.025Z 6 120-139 122 122 122 210 210 210 F F F
132852024-02-20T22:46:55.025Z 7 140-159 119 119 119 206 206 206 F F F
132862024-02-20T22:46:55.025Z 8 160-179 119 119 119 206 206 206 F F F
132872024-02-20T22:46:55.026Z 9 180-199 123 123 123 212 212 212 F F F
132882024-02-20T22:46:55.026Z 10 200-219 118 118 118 205 205 205 F F F
132892024-02-20T22:46:55.026Z 11 220-239 121 121 121 209 209 209 F F F
132902024-02-20T22:46:55.026Z 12 240-259 120 120 120 207 207 207 F F F
132912024-02-20T22:46:55.026Z 13 260-279 121 121 121 209 209 209 F F F
132922024-02-20T22:46:55.026Z 14 280-299 122 122 122 211 211 211 F F F
132932024-02-20T22:46:55.026Z 15 300-319 123 123 123 212 212 212 F F F
132942024-02-20T22:46:55.026Z 16 320-339 123 123 123 212 212 212 F F F
132952024-02-20T22:46:55.026Z 17 340-359 117 117 117 203 203 203 F F F
132962024-02-20T22:46:55.026Z 18 360-379 121 121 121 209 209 209 F F F
132972024-02-20T22:46:55.026Z 19 380-399 121 121 121 209 209 209 F F F
132982024-02-20T22:46:55.026Z 20 400-419 122 122 122 210 210 210 F F F
132992024-02-20T22:46:55.026Z 21 420-439 112 112 112 197 197 197 F F F
133002024-02-20T22:46:55.026Z 22 440-459 123 123 123 212 212 212 F F F
133012024-02-20T22:46:55.027Z 23 460-479 123 123 123 212 212 212 F F F
133022024-02-20T22:46:55.027Z 24 480-499 121 121 121 209 209 209 F F F
133032024-02-20T22:46:55.027Z 25 500-519 117 117 117 203 203 203 F F F
133042024-02-20T22:46:55.027Z 26 520-539 123 123 123 212 212 212 F F F
133052024-02-20T22:46:55.027Z 27 540-559 117 117 117 203 203 203 F F F
133062024-02-20T22:46:55.027Z 28 560-579 123 123 123 212 212 212 F F F
133072024-02-20T22:46:55.027Z 29 580-599 123 123 123 212 212 212 F F F
133082024-02-20T22:46:55.027ZMax gen: 123, Max flush: 212
133092024-02-20T22:46:55.027Z
133102024-02-20T22:46:55.027ZBegin loop 62
133112024-02-20T22:46:55.027ZDownstairs to restart: 0
133122024-02-20T22:46:55.027Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.151737773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634}
133132024-02-20T22:46:55.027Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152039252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634}
133142024-02-20T22:46:55.028Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152056791Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634}
133152024-02-20T22:46:55.028Z{"msg":"Crucible 734eabbb-b2f5-4c66-8d2e-5122e6a415d3 has session id: c41e5913-c1e0-4e2b-b506-6187946e4d20","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.15208948Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133162024-02-20T22:46:55.028ZCrucible runtime is spawned
133172024-02-20T22:46:55.028Z{"msg":"Upstairs opts: Upstairs UUID: 734eabbb-b2f5-4c66-8d2e-5122e6a415d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152153008Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133182024-02-20T22:46:55.028Z{"msg":"Crucible stats registered with UUID: 734eabbb-b2f5-4c66-8d2e-5122e6a415d3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152181667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133192024-02-20T22:46:55.028Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152203896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634}
133202024-02-20T22:46:55.028Z{"msg":"734eabbb-b2f5-4c66-8d2e-5122e6a415d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152358231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133212024-02-20T22:46:55.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152404839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133222024-02-20T22:46:55.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152430318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133232024-02-20T22:46:55.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152471867Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133242024-02-20T22:46:55.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152511755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133252024-02-20T22:46:55.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152576313Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133262024-02-20T22:46:55.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152634861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133272024-02-20T22:46:55.029Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152694888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133282024-02-20T22:46:55.029Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152744067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133292024-02-20T22:46:55.029Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152782145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133302024-02-20T22:46:55.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152808624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133312024-02-20T22:46:55.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152846543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133322024-02-20T22:46:55.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.152888412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133332024-02-20T22:46:55.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.153423873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133342024-02-20T22:46:55.029Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.153466251Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133352024-02-20T22:46:55.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.1534898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133362024-02-20T22:46:55.029Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.153522229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133372024-02-20T22:46:55.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.153562837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133382024-02-20T22:46:55.029Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.153617375Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133392024-02-20T22:46:55.030Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154068429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133402024-02-20T22:46:55.030Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154120137Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133412024-02-20T22:46:55.030Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154359219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133422024-02-20T22:46:55.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154592341Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133432024-02-20T22:46:55.030Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154644229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133442024-02-20T22:46:55.030Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154672168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133452024-02-20T22:46:55.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154696187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133462024-02-20T22:46:55.030Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154738365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133472024-02-20T22:46:55.030Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154758545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133482024-02-20T22:46:55.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154792344Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133492024-02-20T22:46:55.030Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154810403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133502024-02-20T22:46:55.030Z{"msg":"[0]R flush_numbers[0..12]: [209, 210, 212, 212, 212, 209, 210, 206, 206, 212, 205, 209]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154832982Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133512024-02-20T22:46:55.030Z{"msg":"[0]R generation[0..12]: [121, 122, 123, 123, 123, 121, 122, 119, 119, 123, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154847411Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133522024-02-20T22:46:55.030Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154868221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133532024-02-20T22:46:55.031Z{"msg":"[1]R flush_numbers[0..12]: [209, 210, 212, 212, 212, 209, 210, 206, 206, 212, 205, 209]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.15488282Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133542024-02-20T22:46:55.031Z{"msg":"[1]R generation[0..12]: [121, 122, 123, 123, 123, 121, 122, 119, 119, 123, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154916359Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133552024-02-20T22:46:55.031Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.154935748Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133562024-02-20T22:46:55.031Z{"msg":"[2]R flush_numbers[0..12]: [209, 210, 212, 212, 212, 209, 210, 206, 206, 212, 205, 209]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
133572024-02-20T22:46:55.031Z30,"time":"2024-02-20T22:46:54.154975777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133582024-02-20T22:46:55.031Z{"msg":"[2]R generation[0..12]: [121, 122, 123, 123, 123, 121, 122, 119, 119, 123, 118, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155013865Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133592024-02-20T22:46:55.031Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155067934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133602024-02-20T22:46:55.031Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155087753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133612024-02-20T22:46:55.031Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155132131Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133622024-02-20T22:46:55.031Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.15516448Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133632024-02-20T22:46:55.031Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.15517971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133642024-02-20T22:46:55.032Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155209458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133652024-02-20T22:46:55.032Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155245347Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133662024-02-20T22:46:55.032Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155273866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133672024-02-20T22:46:55.032Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155288756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133682024-02-20T22:46:55.032Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155321954Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133692024-02-20T22:46:55.032Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155356673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133702024-02-20T22:46:55.032Z{"msg":"734eabbb-b2f5-4c66-8d2e-5122e6a415d3 is now active with session: c41e5913-c1e0-4e2b-b506-6187946e4d20","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155382252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133712024-02-20T22:46:55.032Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155401352Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634,"session_id":"c41e5913-c1e0-4e2b-b506-6187946e4d20"}
133722024-02-20T22:46:55.032Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.155434441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634}
133732024-02-20T22:46:55.032ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133742024-02-20T22:46:55.032ZLoading write count information from file "/tmp/test_repair_verify.data"
133752024-02-20T22:46:55.032ZRun Repair workload
133762024-02-20T22:46:55.032Z01/30 Flush
133772024-02-20T22:46:55.039Z02/30 Write block 231 len 16384 data: 16 17 19 19
133782024-02-20T22:46:55.039Z03/30 Write block 507 len 20480 data: 9 8 8 9 12
133792024-02-20T22:46:55.039Z04/30 Read block 196 len 40960
133802024-02-20T22:46:55.039Z05/30 Read block 547 len 36864
133812024-02-20T22:46:55.039Z06/30 Read block 545 len 24576
133822024-02-20T22:46:55.040Z07/30 Read block 518 len 4096
133832024-02-20T22:46:55.040Z08/30 Read block 581 len 36864
133842024-02-20T22:46:55.040Z09/30 Read block 313 len 8192
133852024-02-20T22:46:55.040Z10/30 Read block 192 len 32768
133862024-02-20T22:46:55.040Z11/30 Write block 101 len 40960 data: 7 6 4 6 6 6 6 8 8 11
133872024-02-20T22:46:55.040Z12/30 Read block 521 len 4096
133882024-02-20T22:46:55.040Z13/30 Read block 281 len 16384
133892024-02-20T22:46:55.040Z14/30 Write block 324 len 4096 data: 8
133902024-02-20T22:46:55.040Z15/30 Read block 360 len 36864
133912024-02-20T22:46:55.040Z16/30 Flush
133922024-02-20T22:46:55.040Z17/30 Write block 509 len 16384 data: 9 10 13 9
133932024-02-20T22:46:55.040Z18/30 Read block 497 len 40960
133942024-02-20T22:46:55.040Z19/30 Read block 218 len 28672
133952024-02-20T22:46:55.041Z20/30 Write block 268 len 40960 data: 11 10 8 7 6 4 4 6 5 6
133962024-02-20T22:46:55.041Z21/30 Write block 179 len 36864 data: 13 13 14 13 10 11 10 10 11
133972024-02-20T22:46:55.041Z22/30 Read block 472 len 12288
133982024-02-20T22:46:55.041Z23/30 Flush
133992024-02-20T22:46:55.041Z24/30 Write block 359 len 16384 data: 10 12 11 13
134002024-02-20T22:46:55.041Z25/30 Write block 67 len 4096 data: 12
134012024-02-20T22:46:55.041Z26/30 Read block 218 len 20480
134022024-02-20T22:46:55.041Z27/30 Read block 552 len 16384
134032024-02-20T22:46:55.041Z28/30 Read block 199 len 36864
134042024-02-20T22:46:55.041Z29/30 Read block 414 len 16384
134052024-02-20T22:46:55.041Z30/30 Write block 356 len 4096 data: 7
134062024-02-20T22:46:55.041Z----------------------------------------------------------------
134072024-02-20T22:46:55.041Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134082024-02-20T22:46:55.041ZStates: Active Active Active
134092024-02-20T22:46:55.042ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134102024-02-20T22:46:55.042Z 1 Acked 1000 Flush 0 Sent Done Done false
134112024-02-20T22:46:55.042Z 2 Acked 1001 Write 4 Sent Done Done false
134122024-02-20T22:46:55.042Z 3 Acked 1002 Write 5 Sent Done Done false
134132024-02-20T22:46:55.042Z 4 Acked 1003 Read 10 Sent Done Done false
134142024-02-20T22:46:55.042Z 5 Acked 1004 Read 9 Sent Done Done false
134152024-02-20T22:46:55.042Z 6 Acked 1005 Read 6 Sent Done Done false
134162024-02-20T22:46:55.042Z 7 Acked 1006 Read 1 Sent Done Done false
134172024-02-20T22:46:55.042Z 8 Acked 1007 Read 9 Sent Done Done false
134182024-02-20T22:46:55.042Z 9 Acked 1008 Read 2 Sent Done Done false
134192024-02-20T22:46:55.042Z 10 Acked 1009 Read 8 Sent Done Done false
134202024-02-20T22:46:55.042Z 11 Acked 1010 Write 10 Sent Done Done false
134212024-02-20T22:46:55.042Z 12 Acked 1011 Read 1 Sent Done Done false
134222024-02-20T22:46:55.043Z 13 Acked 1012 Read 4 Sent Done Done false
134232024-02-20T22:46:55.043Z 14 Acked 1013 Write 1 Sent Done Done false
134242024-02-20T22:46:55.043Z 15 Acked 1014 Read 9 Sent Done Done false
134252024-02-20T22:46:55.043Z 16 Acked 1015 Flush 0 Sent Done Done false
134262024-02-20T22:46:55.043Z 17 Acked 1016 Write 4 Sent Done Done false
134272024-02-20T22:46:55.043Z 18 Acked 1017 Read 10 Sent Done Done false
134282024-02-20T22:46:55.043Z 19 Acked 1018 Read 7 Sent Done Done false
134292024-02-20T22:46:55.043Z 20 Acked 1019 Write 10 Sent Done Done false
134302024-02-20T22:46:55.043Z 21 Acked 1020 Write 9 Sent Done Done false
134312024-02-20T22:46:55.043Z 22 Acked 1021 Read 3 Sent Done Done false
134322024-02-20T22:46:55.043Z 23 Acked 1022 Flush 0 Sent Done Done false
134332024-02-20T22:46:55.043Z 24 Acked 1023 Write 4 Sent Done Done false
134342024-02-20T22:46:55.044Z 25 Acked 1024 Write 1 Sent Done Done false
134352024-02-20T22:46:55.044Z 26 Acked 1025 Read 5 Sent Done Done false
134362024-02-20T22:46:55.044Z 27 Acked 1026 Read 4 Sent Done Done false
134372024-02-20T22:46:55.044Z 28 Acked 1027 Read 9 Sent Done Done false
134382024-02-20T22:46:55.044Z 29 Acked 1028 Read 4 Sent Done Done false
134392024-02-20T22:46:55.044Z 30 Acked 1029 Write 1 Sent Sent Sent false
134402024-02-20T22:46:55.044Z STATES DS:0 DS:1 DS:2 TOTAL
134412024-02-20T22:46:55.044Z New 0 0 0 0
134422024-02-20T22:46:55.044Z Sent 30 1 1 32
134432024-02-20T22:46:55.044Z Done 0 29 29 58
134442024-02-20T22:46:55.044Z Skipped 0 0 0 0
134452024-02-20T22:46:55.044Z Error 0 0 0 0
134462024-02-20T22:46:55.044ZLast Flush: 0 1022 1022
134472024-02-20T22:46:55.045ZDownstairs last five completed:
134482024-02-20T22:46:55.045ZUpstairs last five completed: 30 29 28 27 26
134492024-02-20T22:46:55.045Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:54.186808378Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1634}
134502024-02-20T22:46:55.045ZWrote out file "/tmp/test_repair_verify.data"
134512024-02-20T22:46:55.045Z
134522024-02-20T22:46:55.045ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134532024-02-20T22:46:55.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134542024-02-20T22:46:55.045Z 0 000-019 121 121 121 209 209 209 F F F
134552024-02-20T22:46:55.045Z 1 020-039 122 122 122 210 210 210 F F F
134562024-02-20T22:46:55.045Z 2 040-059 123 123 123 212 212 212 F F F
134572024-02-20T22:46:55.045Z 3 060-079 123 123 123 212 212 212 F T T <---
134582024-02-20T22:46:55.045Z 4 080-099 123 123 123 212 212 212 F F F
134592024-02-20T22:46:55.046Z 5 100-119 121 124 124 209 214 214 F F F <---
134602024-02-20T22:46:55.046Z 6 120-139 122 122 122 210 210 210 F F F
134612024-02-20T22:46:55.046Z 7 140-159 119 119 119 206 206 206 F F F
134622024-02-20T22:46:55.046Z 8 160-179 119 124 124 206 215 215 F F F <---
134632024-02-20T22:46:55.046Z 9 180-199 123 124 124 212 215 215 F F F <---
134642024-02-20T22:46:55.046Z 10 200-219 118 118 118 205 205 205 F F F
134652024-02-20T22:46:55.046Z 11 220-239 121 124 124 209 214 214 F F F <---
134662024-02-20T22:46:55.046Z 12 240-259 120 120 120 207 207 207 F F F
134672024-02-20T22:46:55.046Z 13 260-279 121 124 124 209 215 215 F F F <---
134682024-02-20T22:46:55.046Z 14 280-299 122 122 122 211 211 211 F F F
134692024-02-20T22:46:55.046Z 15 300-319 123 123 123 212 212 212 F F F
134702024-02-20T22:46:55.046Z 16 320-339 123 124 124 212 214 214 F F F <---
134712024-02-20T22:46:55.046Z 17 340-359 117 117 117 203 203 203 F T T <---
134722024-02-20T22:46:55.047Z 18 360-379 121 121 121 209 209 209 F T T <---
134732024-02-20T22:46:55.047Z 19 380-399 121 121 121 209 209 209 F F F
134742024-02-20T22:46:55.047Z 20 400-419 122 122 122 210 210 210 F F F
134752024-02-20T22:46:55.047Z 21 420-439 112 112 112 197 197 197 F F F
134762024-02-20T22:46:55.047Z 22 440-459 123 123 123 212 212 212 F F F
134772024-02-20T22:46:55.052Z 23 460-479 123 123 123 212 212 212 F F F
134782024-02-20T22:46:55.053Z 24 480-499 121 121 121 209 209 209 F F F
134792024-02-20T22:46:55.053Z 25 500-519 117 124 124 203 215 215 F F F <---
134802024-02-20T22:46:55.053Z 26 520-539 123 123 123 212 212 212 F F F
134812024-02-20T22:46:55.053Z 27 540-559 117 117 117 203 203 203 F F F
134822024-02-20T22:46:55.053Z 28 560-579 123 123 123 212 212 212 F F F
134832024-02-20T22:46:55.053Z 29 580-599 123 123 123 212 212 212 F F F
134842024-02-20T22:46:55.053ZMax gen: 124, Max flush: 215
134852024-02-20T22:46:55.053ZError: Difference in extent metadata found!
134862024-02-20T22:46:55.053ZA Difference in extent metadata is expected here
134872024-02-20T22:46:55.053ZOn loop 62
134882024-02-20T22:46:55.053Z
134892024-02-20T22:46:55.053ZVerifying data now
134902024-02-20T22:46:55.222ZNo change after verify
134912024-02-20T22:46:55.222ZLoop: 62 Downstairs dump after verify (and repair):
134922024-02-20T22:46:55.236ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134932024-02-20T22:46:55.236Z 0 000-019 121 121 121 209 209 209 F F F
134942024-02-20T22:46:55.236Z 1 020-039 122 122 122 210 210 210 F F F
134952024-02-20T22:46:55.236Z 2 040-059 123 123 123 212 212 212 F F F
134962024-02-20T22:46:55.236Z 3 060-079 125 125 125 216 216 216 F F F
134972024-02-20T22:46:55.236Z 4 080-099 123 123 123 212 212 212 F F F
134982024-02-20T22:46:55.236Z 5 100-119 124 124 124 214 214 214 F F F
134992024-02-20T22:46:55.236Z 6 120-139 122 122 122 210 210 210 F F F
135002024-02-20T22:46:55.236Z 7 140-159 119 119 119 206 206 206 F F F
135012024-02-20T22:46:55.241Z 8 160-179 124 124 124 215 215 215 F F F
135022024-02-20T22:46:55.241Z 9 180-199 124 124 124 215 215 215 F F F
135032024-02-20T22:46:55.241Z 10 200-219 118 118 118 205 205 205 F F F
135042024-02-20T22:46:55.241Z 11 220-239 124 124 124 214 214 214 F F F
135052024-02-20T22:46:55.241Z 12 240-259 120 120 120 207 207 207 F F F
135062024-02-20T22:46:55.241Z 13 260-279 124 124 124 215 215 215 F F F
135072024-02-20T22:46:55.241Z 14 280-299 122 122 122 211 211 211 F F F
135082024-02-20T22:46:55.241Z 15 300-319 123 123 123 212 212 212 F F F
135092024-02-20T22:46:55.241Z 16 320-339 124 124 124 214 214 214 F F F
135102024-02-20T22:46:55.241Z 17 340-359 125 125 125 216 216 216 F F F
135112024-02-20T22:46:55.241Z 18 360-379 125 125 125 216 216 216 F F F
135122024-02-20T22:46:55.241Z 19 380-399 121 121 121 209 209 209 F F F
135132024-02-20T22:46:55.241Z 20 400-419 122 122 122 210 210 210 F F F
135142024-02-20T22:46:55.241Z 21 420-439 112 112 112 197 197 197 F F F
135152024-02-20T22:46:55.241Z 22 440-459 123 123 123 212 212 212 F F F
135162024-02-20T22:46:55.241Z 23 460-479 123 123 123 212 212 212 F F F
135172024-02-20T22:46:55.241Z 24 480-499 121 121 121 209 209 209 F F F
135182024-02-20T22:46:55.241Z 25 500-519 124 124 124 215 215 215 F F F
135192024-02-20T22:46:55.241Z 26 520-539 123 123 123 212 212 212 F F F
135202024-02-20T22:46:55.241Z 27 540-559 117 117 117 203 203 203 F F F
135212024-02-20T22:46:55.241Z 28 560-579 123 123 123 212 212 212 F F F
135222024-02-20T22:46:55.241Z 29 580-599 123 123 123 212 212 212 F F F
135232024-02-20T22:46:55.242ZMax gen: 125, Max flush: 216
135242024-02-20T22:46:55.242Z
135252024-02-20T22:46:55.242ZBegin loop 63
135262024-02-20T22:46:55.242ZDownstairs to restart: 0
135272024-02-20T22:46:55.259Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775348208Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642}
135282024-02-20T22:46:55.259Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775635288Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642}
135292024-02-20T22:46:55.270Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775673766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642}
135302024-02-20T22:46:55.270Z{"msg":"Crucible aee11888-9e1e-4d68-b2a9-45dfd1ba0873 has session id: a5dcab27-fe0d-40aa-9ea1-d1864fa9e231","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775707655Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135312024-02-20T22:46:55.270Z{Crucible runtime is spawned
135322024-02-20T22:46:55.270Z"msg":"Upstairs opts: Upstairs UUID: aee11888-9e1e-4d68-b2a9-45dfd1ba0873, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775758633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135332024-02-20T22:46:55.271Z{"msg":"Crucible stats registered with UUID: aee11888-9e1e-4d68-b2a9-45dfd1ba0873","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775794222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135342024-02-20T22:46:55.271Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775816531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642}
135352024-02-20T22:46:55.271Z{"msg":"aee11888-9e1e-4d68-b2a9-45dfd1ba0873 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77585515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135362024-02-20T22:46:55.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775870179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135372024-02-20T22:46:55.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775886389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135382024-02-20T22:46:55.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775900318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135392024-02-20T22:46:55.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775914048Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135402024-02-20T22:46:55.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775936677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135412024-02-20T22:46:55.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.775960656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135422024-02-20T22:46:55.271Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776021714Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135432024-02-20T22:46:55.271Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776069752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135442024-02-20T22:46:55.271Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776106261Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135452024-02-20T22:46:55.271Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77614735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135462024-02-20T22:46:55.271Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776300474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135472024-02-20T22:46:55.271Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776349402Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135482024-02-20T22:46:55.271Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776908832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135492024-02-20T22:46:55.271Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.776945571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135502024-02-20T22:46:55.271Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77697493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135512024-02-20T22:46:55.271Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.777013159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135522024-02-20T22:46:55.271Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.777035238Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135532024-02-20T22:46:55.271Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.777100245Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135542024-02-20T22:46:55.271Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.777563429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135552024-02-20T22:46:55.271Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.777665105Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135562024-02-20T22:46:55.271Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.777701224Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135572024-02-20T22:46:55.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77808803Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135582024-02-20T22:46:55.272Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778138308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135592024-02-20T22:46:55.272Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778194466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135602024-02-20T22:46:55.272Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778241024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135612024-02-20T22:46:55.272Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778259724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135622024-02-20T22:46:55.272Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778281033Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135632024-02-20T22:46:55.272Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778295902Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135642024-02-20T22:46:55.272Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778309672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135652024-02-20T22:46:55.272Z{"msg":"[0]R flush_numbers[0..12]: [209, 210, 212, 216, 212, 214, 210, 206, 215, 215, 205, 214]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778331811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135662024-02-20T22:46:55.272Z{"msg":"[0]R generation[0..12]: [121, 122, 123, 125, 123, 124, 122, 119, 124, 124, 118, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778346991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135672024-02-20T22:46:55.272Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77836203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135682024-02-20T22:46:55.272Z{"msg":"[1]R flush_numbers[0..12]: [209, 210, 212, 216, 212, 214, 210, 206, 215, 215, 205, 214]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77837632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135692024-02-20T22:46:55.272Z{"msg":"[1]R generation[0..12]: [121, 122, 123, 125, 123, 124, 122, 119, 124, 124, 118, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778397019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135702024-02-20T22:46:55.272Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778412168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135712024-02-20T22:46:55.272Z{"msg":"[2]R flush_numbers[0..12]: [209, 210, 212, 216, 212, 214, 210, 206, 215, 215, 205, 214]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778427218Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135722024-02-20T22:46:55.272Z{"msg":"[2]R generation[0..12]: [121, 122, 123, 125, 123, 124, 122, 119, 124, 124, 118, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778441037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135732024-02-20T22:46:55.272Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778462476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135742024-02-20T22:46:55.272Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778477546Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135752024-02-20T22:46:55.272Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778504185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135762024-02-20T22:46:55.272Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778518654Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135772024-02-20T22:46:55.272Z{"msgWait for a query_work_queue command to finish before sending IO
135782024-02-20T22:46:55.272Z":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778543504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135792024-02-20T22:46:55.272Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778564183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135802024-02-20T22:46:55.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778576423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135812024-02-20T22:46:55.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778603131Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135822024-02-20T22:46:55.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778633861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135832024-02-20T22:46:55.273Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.77865073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135842024-02-20T22:46:55.273Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778671169Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135852024-02-20T22:46:55.273Z{"msg":"aee11888-9e1e-4d68-b2a9-45dfd1ba0873 is now active with session: a5dcab27-fe0d-40aa-9ea1-d1864fa9e231","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778685189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135862024-02-20T22:46:55.273Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778714118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642,"session_id":"a5dcab27-fe0d-40aa-9ea1-d1864fa9e231"}
135872024-02-20T22:46:55.273Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:54.778739017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642}
135882024-02-20T22:46:55.273ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135892024-02-20T22:46:55.273ZLoading write count information from file "/tmp/test_repair_verify.data"
135902024-02-20T22:46:55.273ZRun Repair workload
135912024-02-20T22:46:55.273Z01/30 Write block 72 len 8192 data: 12 10
135922024-02-20T22:46:55.273Z02/30 Write block 142 len 8192 data: 9 11
135932024-02-20T22:46:55.273Z03/30 Flush
135942024-02-20T22:46:55.278Z04/30 Write block 476 len 40960 data: 12 16 17 13 16 15 15 15 13 11
135952024-02-20T22:46:55.278Z05/30 Read block 244 len 28672
135962024-02-20T22:46:55.278Z06/30 Read block 224 len 28672
135972024-02-20T22:46:55.278Z07/30 Write block 246 len 12288 data: 5 9 8
135982024-02-20T22:46:55.278Z08/30 Read block 347 len 16384
135992024-02-20T22:46:55.278Z09/30 Read block 475 len 20480
136002024-02-20T22:46:55.278Z10/30 Flush
136012024-02-20T22:46:55.278Z11/30 Write block 228 len 12288 data: 10 14 18
136022024-02-20T22:46:55.278Z12/30 Write block 4 len 8192 data: 10 10
136032024-02-20T22:46:55.278Z13/30 Write block 210 len 28672 data: 9 9 8 4 4 5 5
136042024-02-20T22:46:55.278Z14/30 Read block 114 len 32768
136052024-02-20T22:46:55.283Z15/30 Flush
136062024-02-20T22:46:55.283Z16/30 Write block 271 len 28672 data: 8 7 5 5 7 6 7
136072024-02-20T22:46:55.283Z17/30 Flush
136082024-02-20T22:46:55.287Z18/30 Flush
136092024-02-20T22:46:55.287Z19/30 Write block 90 len 16384 data: 8 8 5 5
136102024-02-20T22:46:55.287Z20/30 Write block 246 len 28672 data: 6 10 9 6 7 7 7
136112024-02-20T22:46:55.287Z21/30 Write block 478 len 40960 data: 18 14 17 16 16 16 14 12 9 8
136122024-02-20T22:46:55.293Z22/30 Write block 380 len 24576 data: 9 8 10 8 9 7
136132024-02-20T22:46:55.293Z23/30 Flush
136142024-02-20T22:46:55.297Z24/30 Write block 232 len 40960 data: 18 20 20 16 16 15 9 8 6 5
136152024-02-20T22:46:55.297Z25/30 Read block 342 len 24576
136162024-02-20T22:46:55.297Z26/30 Flush
136172024-02-20T22:46:55.297Z27/30 Write block 265 len 12288 data: 11 12 12
136182024-02-20T22:46:55.297Z28/30 Read block 344 len 36864
136192024-02-20T22:46:55.297Z29/30 Write block 60 len 28672 data: 7 6 7 8 8 9 8
136202024-02-20T22:46:55.298Z30/30 Read block 399 len 24576
136212024-02-20T22:46:55.303Z----------------------------------------------------------------
136222024-02-20T22:46:55.303Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136232024-02-20T22:46:55.303ZStates: Active Active Active
136242024-02-20T22:46:55.303ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136252024-02-20T22:46:55.303Z 1 Acked 1000 Write 2 Done Done Done false
136262024-02-20T22:46:55.303Z 2 Acked 1001 Write 2 Sent Done Done false
136272024-02-20T22:46:55.303Z 3 Acked 1002 Flush 0 Sent Done Done false
136282024-02-20T22:46:55.303Z 4 Acked 1003 Write 10 Sent Done Done false
136292024-02-20T22:46:55.303Z 5 Acked 1004 Read 7 Sent Done Done false
136302024-02-20T22:46:55.303Z 6 Acked 1005 Read 7 Sent Done Done false
136312024-02-20T22:46:55.303Z 7 Acked 1006 Write 3 Sent Done Done false
136322024-02-20T22:46:55.303Z 8 Acked 1007 Read 4 Sent Done Done false
136332024-02-20T22:46:55.303Z 9 Acked 1008 Read 5 Sent Done Done false
136342024-02-20T22:46:55.303Z 10 Acked 1009 Flush 0 Sent Done Done false
136352024-02-20T22:46:55.303Z 11 Acked 1010 Write 3 Sent Done Done false
136362024-02-20T22:46:55.303Z 12 Acked 1011 Write 2 Sent Done Done false
136372024-02-20T22:46:55.303Z 13 Acked 1012 Write 7 Sent Done Done false
136382024-02-20T22:46:55.303Z 14 Acked 1013 Read 8 Sent Done Done false
136392024-02-20T22:46:55.303Z 15 Acked 1014 Flush 0 Sent Done Done false
136402024-02-20T22:46:55.303Z 16 Acked 1015 Write 7 Sent Done Done false
136412024-02-20T22:46:55.303Z 17 Acked 1016 Flush 0 Sent Done Done false
136422024-02-20T22:46:55.303Z 18 Acked 1017 Flush 0 Sent Done Done false
136432024-02-20T22:46:55.303Z 19 Acked 1018 Write 4 Sent Done Done false
136442024-02-20T22:46:55.303Z 20 Acked 1019 Write 7 Sent Done Done false
136452024-02-20T22:46:55.304Z 21 Acked 1020 Write 10 Sent Done Done false
136462024-02-20T22:46:55.304Z 22 Acked 1021 Write 6 Sent Done Done false
136472024-02-20T22:46:55.304Z 23 Acked 1022 Flush 0 Sent Done Done false
136482024-02-20T22:46:55.304Z 24 Acked 1023 Write 10 Sent Done Done false
136492024-02-20T22:46:55.304Z 25 Acked 1024 Read 6 Sent Done Done false
136502024-02-20T22:46:55.304Z 26 Acked 1025 Flush 0 Sent Done Done false
136512024-02-20T22:46:55.304Z 27 Acked 1026 Write 3 Sent Done Done false
136522024-02-20T22:46:55.304Z 28 Acked 1027 Read 9 Sent Done Done false
136532024-02-20T22:46:55.304Z 29 Acked 1028 Write 7 Sent Done Done false
136542024-02-20T22:46:55.304Z 30 Acked 1029 Read 6 Sent Done Done false
136552024-02-20T22:46:55.304Z STATES DS:0 DS:1 DS:2 TOTAL
136562024-02-20T22:46:55.304Z New 0 0 0 0
136572024-02-20T22:46:55.304Z Sent 29 0 0 29
136582024-02-20T22:46:55.304Z Done 1 30 30 61
136592024-02-20T22:46:55.304Z Skipped 0 0 0 0
136602024-02-20T22:46:55.304Z Error 0 0 0 0
136612024-02-20T22:46:55.304ZLast Flush: 0 1025 1025
136622024-02-20T22:46:55.304ZDownstairs last five completed:
136632024-02-20T22:46:55.304ZUpstairs last five completed: 30 29 28 27 26
136642024-02-20T22:46:55.304Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:54.816011993Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1642}
136652024-02-20T22:46:55.304ZWrote out file "/tmp/test_repair_verify.data"
136662024-02-20T22:46:55.308Z
136672024-02-20T22:46:55.308ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136682024-02-20T22:46:55.322ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136692024-02-20T22:46:55.322Z 0 000-019 121 126 126 209 219 219 F F F <---
136702024-02-20T22:46:55.327Z 1 020-039 122 122 122 210 210 210 F F F
136712024-02-20T22:46:55.327Z 2 040-059 123 123 123 212 212 212 F F F
136722024-02-20T22:46:55.328Z 3 060-079 125 126 126 216 217 217 T T T <---
136732024-02-20T22:46:55.328Z 4 080-099 123 126 126 212 222 222 F F F <---
136742024-02-20T22:46:55.328Z 5 100-119 124 124 124 214 214 214 F F F
136752024-02-20T22:46:55.328Z 6 120-139 122 122 122 210 210 210 F F F
136762024-02-20T22:46:55.328Z 7 140-159 119 126 126 206 217 217 F F F <---
136772024-02-20T22:46:55.328Z 8 160-179 124 124 124 215 215 215 F F F
136782024-02-20T22:46:55.328Z 9 180-199 124 124 124 215 215 215 F F F
136792024-02-20T22:46:55.328Z 10 200-219 118 126 126 205 219 219 F F F <---
136802024-02-20T22:46:55.328Z 11 220-239 124 126 126 214 223 223 F F F <---
136812024-02-20T22:46:55.328Z 12 240-259 120 126 126 207 223 223 F F F <---
136822024-02-20T22:46:55.328Z 13 260-279 124 126 126 215 220 220 F T T <---
136832024-02-20T22:46:55.328Z 14 280-299 122 122 122 211 211 211 F F F
136842024-02-20T22:46:55.328Z 15 300-319 123 123 123 212 212 212 F F F
136852024-02-20T22:46:55.328Z 16 320-339 124 124 124 214 214 214 F F F
136862024-02-20T22:46:55.328Z 17 340-359 125 125 125 216 216 216 F F F
136872024-02-20T22:46:55.328Z 18 360-379 125 125 125 216 216 216 F F F
136882024-02-20T22:46:55.328Z 19 380-399 121 126 126 209 222 222 F F F <---
136892024-02-20T22:46:55.328Z 20 400-419 122 122 122 210 210 210 F F F
136902024-02-20T22:46:55.328Z 21 420-439 112 112 112 197 197 197 F F F
136912024-02-20T22:46:55.328Z 22 440-459 123 123 123 212 212 212 F F F
136922024-02-20T22:46:55.328Z 23 460-479 123 126 126 212 222 222 F F F <---
136932024-02-20T22:46:55.328Z 24 480-499 121 126 126 209 222 222 F F F <---
136942024-02-20T22:46:55.328Z 25 500-519 124 124 124 215 215 215 F F F
136952024-02-20T22:46:55.328Z 26 520-539 123 123 123 212 212 212 F F F
136962024-02-20T22:46:55.328Z 27 540-559 117 117 117 203 203 203 F F F
136972024-02-20T22:46:55.329Z 28 560-579 123 123 123 212 212 212 F F F
136982024-02-20T22:46:55.329Z 29 580-599 123 123 123 212 212 212 F F F
136992024-02-20T22:46:55.329ZMax gen: 126, Max flush: 223
137002024-02-20T22:46:55.329ZError: Difference in extent metadata found!
137012024-02-20T22:46:55.329ZA Difference in extent metadata is expected here
137022024-02-20T22:46:55.329ZOn loop 63
137032024-02-20T22:46:55.329Z
137042024-02-20T22:46:55.333ZVerifying data now
137052024-02-20T22:46:59.883ZNo change after verify
137062024-02-20T22:46:59.883ZLoop: 63 Downstairs dump after verify (and repair):
137072024-02-20T22:46:59.897ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137082024-02-20T22:46:59.897Z 0 000-019 126 126 126 219 219 219 F F F
137092024-02-20T22:46:59.897Z 1 020-039 122 122 122 210 210 210 F F F
137102024-02-20T22:46:59.897Z 2 040-059 123 123 123 212 212 212 F F F
137112024-02-20T22:46:59.897Z 3 060-079 127 127 127 224 224 224 F F F
137122024-02-20T22:46:59.897Z 4 080-099 126 126 126 222 222 222 F F F
137132024-02-20T22:46:59.897Z 5 100-119 124 124 124 214 214 214 F F F
137142024-02-20T22:46:59.897Z 6 120-139 122 122 122 210 210 210 F F F
137152024-02-20T22:46:59.904Z 7 140-159 126 126 126 217 217 217 F F F
137162024-02-20T22:46:59.905Z 8 160-179 124 124 124 215 215 215 F F F
137172024-02-20T22:46:59.905Z 9 180-199 124 124 124 215 215 215 F F F
137182024-02-20T22:46:59.905Z 10 200-219 126 126 126 219 219 219 F F F
137192024-02-20T22:46:59.905Z 11 220-239 126 126 126 223 223 223 F F F
137202024-02-20T22:46:59.905Z 12 240-259 126 126 126 223 223 223 F F F
137212024-02-20T22:46:59.905Z 13 260-279 127 127 127 224 224 224 F F F
137222024-02-20T22:46:59.905Z 14 280-299 122 122 122 211 211 211 F F F
137232024-02-20T22:46:59.905Z 15 300-319 123 123 123 212 212 212 F F F
137242024-02-20T22:46:59.905Z 16 320-339 124 124 124 214 214 214 F F F
137252024-02-20T22:46:59.905Z 17 340-359 125 125 125 216 216 216 F F F
137262024-02-20T22:46:59.905Z 18 360-379 125 125 125 216 216 216 F F F
137272024-02-20T22:46:59.905Z 19 380-399 126 126 126 222 222 222 F F F
137282024-02-20T22:46:59.905Z 20 400-419 122 122 122 210 210 210 F F F
137292024-02-20T22:46:59.905Z 21 420-439 112 112 112 197 197 197 F F F
137302024-02-20T22:46:59.905Z 22 440-459 123 123 123 212 212 212 F F F
137312024-02-20T22:46:59.905Z 23 460-479 126 126 126 222 222 222 F F F
137322024-02-20T22:46:59.905Z 24 480-499 126 126 126 222 222 222 F F F
137332024-02-20T22:46:59.905Z 25 500-519 124 124 124 215 215 215 F F F
137342024-02-20T22:46:59.905Z 26 520-539 123 123 123 212 212 212 F F F
137352024-02-20T22:46:59.905Z 27 540-559 117 117 117 203 203 203 F F F
137362024-02-20T22:46:59.905Z 28 560-579 123 123 123 212 212 212 F F F
137372024-02-20T22:46:59.905Z 29 580-599 123 123 123 212 212 212 F F F
137382024-02-20T22:46:59.905ZMax gen: 127, Max flush: 224
137392024-02-20T22:46:59.906Z
137402024-02-20T22:46:59.906ZBegin loop 64
137412024-02-20T22:46:59.906ZDownstairs to restart: 2
137422024-02-20T22:46:59.919Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435035371Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650}
137432024-02-20T22:46:59.930Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435400348Z","hostname":"Crucible runtime is spawned
137442024-02-20T22:46:59.930Zip-10-150-1-191.us-west-2.compute.internal","pid":1650}
137452024-02-20T22:46:59.931Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435446936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650}
137462024-02-20T22:46:59.931Z{"msg":"Crucible 1facb67b-c706-4ea6-89be-5efe6118d906 has session id: 06703675-46cb-4e62-933e-582c8503bbaf","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435489185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137472024-02-20T22:46:59.931Z{"msg":"Upstairs opts: Upstairs UUID: 1facb67b-c706-4ea6-89be-5efe6118d906, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435543672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137482024-02-20T22:46:59.931Z{"msg":"Crucible stats registered with UUID: 1facb67b-c706-4ea6-89be-5efe6118d906","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435584431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137492024-02-20T22:46:59.931Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.43562619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650}
137502024-02-20T22:46:59.931Z{"msg":"1facb67b-c706-4ea6-89be-5efe6118d906 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435655099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137512024-02-20T22:46:59.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435697047Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137522024-02-20T22:46:59.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435743755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137532024-02-20T22:46:59.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435789144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137542024-02-20T22:46:59.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435839182Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137552024-02-20T22:46:59.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435863941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137562024-02-20T22:46:59.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435922209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137572024-02-20T22:46:59.931Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435951068Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137582024-02-20T22:46:59.931Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.435980777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137592024-02-20T22:46:59.931Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436022775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137602024-02-20T22:46:59.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436062514Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137612024-02-20T22:46:59.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436091123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137622024-02-20T22:46:59.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436128092Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137632024-02-20T22:46:59.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436659983Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137642024-02-20T22:46:59.931Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436703351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137652024-02-20T22:46:59.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.43673131Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137662024-02-20T22:46:59.931Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436766789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137672024-02-20T22:46:59.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436796758Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137682024-02-20T22:46:59.931Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.436823697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137692024-02-20T22:46:59.932Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.43728204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137702024-02-20T22:46:59.932Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.437320519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137712024-02-20T22:46:59.932Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.437521562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137722024-02-20T22:46:59.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.437813311Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137732024-02-20T22:46:59.932Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.4378565Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137742024-02-20T22:46:59.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.437874429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137752024-02-20T22:46:59.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.437960446Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137762024-02-20T22:46:59.932Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438005704Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137772024-02-20T22:46:59.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438024264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137782024-02-20T22:46:59.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438037403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137792024-02-20T22:46:59.932Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438051583Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137802024-02-20T22:46:59.932Z{"msg":"[0]R flush_numbers[0..12]: [219, 210, 212, 224, 222, 214, 210, 217, 215, 215, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438065622Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137812024-02-20T22:46:59.932Z{"msg":"[0]R generation[0..12]: [126, 122, 123, 127, 126, 124, 122, 126, 124, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438088681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137822024-02-20T22:46:59.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438103571Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137832024-02-20T22:46:59.932Z{"msg":"[1]R flush_numbers[0..12]: [219, 210, 212, 224, 222, 214, 210, 217, 215, 215, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438118991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137842024-02-20T22:46:59.932Z{"msg":"[1]R generation[0..12]: [126, 122, 123, 127, 126, 124, 122, 126, 124, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.43813318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137852024-02-20T22:46:59.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438155589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137862024-02-20T22:46:59.932Z{"msg":"[2]R flush_numbers[0..12]: [219, 210, 212, 224, 222, 214, 210, 217, 215, 215, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438170458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137872024-02-20T22:46:59.932Z{"msg":"[2]R generation[0..12]: [126, 122, 123, 127, 126, 124, 122, 126, 124, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438184808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137882024-02-20T22:46:59.932Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438198797Z"Wait for a query_work_queue command to finish before sending IO
137892024-02-20T22:46:59.932Z,"hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137902024-02-20T22:46:59.932Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438226627Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137912024-02-20T22:46:59.932Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438240256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137922024-02-20T22:46:59.932Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438253595Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137932024-02-20T22:46:59.933Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438267415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137942024-02-20T22:46:59.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438280915Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137952024-02-20T22:46:59.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438293784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137962024-02-20T22:46:59.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438307074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137972024-02-20T22:46:59.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438320643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137982024-02-20T22:46:59.933Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438333943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
137992024-02-20T22:46:59.933Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438355002Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
138002024-02-20T22:46:59.933Z{"msg":"1facb67b-c706-4ea6-89be-5efe6118d906 is now active with session: 06703675-46cb-4e62-933e-582c8503bbaf","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438368382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
138012024-02-20T22:46:59.933Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438380411Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650,"session_id":"06703675-46cb-4e62-933e-582c8503bbaf"}
138022024-02-20T22:46:59.933Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:46:59.438392651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650}
138032024-02-20T22:46:59.933ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138042024-02-20T22:46:59.933ZLoading write count information from file "/tmp/test_repair_verify.data"
138052024-02-20T22:46:59.933ZRun Repair workload
138062024-02-20T22:46:59.938Z01/30 Write block 165 len 24576 data: 8 9 6 7 7 10
138072024-02-20T22:46:59.938Z02/30 Write block 359 len 20480 data: 11 13 12 14 16
138082024-02-20T22:46:59.938Z03/30 Read block 383 len 12288
138092024-02-20T22:46:59.938Z04/30 Write block 108 len 28672 data: 9 9 12 12 11 11 10
138102024-02-20T22:46:59.938Z05/30 Read block 188 len 28672
138112024-02-20T22:46:59.938Z06/30 Read block 157 len 28672
138122024-02-20T22:46:59.939Z07/30 Read block 462 len 40960
138132024-02-20T22:46:59.939Z08/30 Write block 338 len 4096 data: 14
138142024-02-20T22:46:59.939Z09/30 Write block 265 len 4096 data: 12
138152024-02-20T22:46:59.939Z10/30 Read block 515 len 32768
138162024-02-20T22:46:59.939Z11/30 Read block 351 len 28672
138172024-02-20T22:46:59.939Z12/30 Write block 541 len 12288 data: 13 10 11
138182024-02-20T22:46:59.939Z13/30 Write block 350 len 28672 data: 10 8 6 6 5 5 8
138192024-02-20T22:46:59.939Z14/30 Flush
138202024-02-20T22:46:59.939Z15/30 Write block 148 len 8192 data: 9 11
138212024-02-20T22:46:59.939Z16/30 Read block 81 len 40960
138222024-02-20T22:46:59.944Z17/30 Read block 204 len 20480
138232024-02-20T22:46:59.944Z18/30 Read block 165 len 12288
138242024-02-20T22:46:59.944Z19/30 Read block 154 len 12288
138252024-02-20T22:46:59.944Z20/30 Write block 425 len 40960 data: 8 7 5 7 7 6 5 4 4 5
138262024-02-20T22:46:59.944Z21/30 Read block 434 len 4096
138272024-02-20T22:46:59.948Z22/30 Write block 545 len 24576 data: 10 10 10 9 9 7
138282024-02-20T22:46:59.948Z23/30 Read block 589 len 4096
138292024-02-20T22:46:59.948Z24/30 Read block 518 len 12288
138302024-02-20T22:46:59.948Z25/30 Write block 296 len 40960 data: 6 8 11 11 10 14 16 16 15 13
138312024-02-20T22:46:59.948Z26/30 Write block 441 len 4096 data: 14
138322024-02-20T22:46:59.948Z27/30 Read block 180 len 32768
138332024-02-20T22:46:59.953Z28/30 Write block 20 len 4096 data: 12
138342024-02-20T22:46:59.953Z29/30 Read block 468 len 8192
138352024-02-20T22:46:59.953Z30/30 Read block 62 len 12288
138362024-02-20T22:46:59.953Z----------------------------------------------------------------
138372024-02-20T22:46:59.953Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138382024-02-20T22:46:59.953ZStates: Active Active Active
138392024-02-20T22:46:59.953ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138402024-02-20T22:46:59.953Z 1 Acked 1000 Write 6 Done Done Sent false
138412024-02-20T22:46:59.953Z 2 Acked 1001 Write 5 Done Done Sent false
138422024-02-20T22:46:59.953Z 3 Acked 1002 Read 3 Done Done Sent false
138432024-02-20T22:46:59.953Z 4 Acked 1003 Write 7 Done Done Sent false
138442024-02-20T22:46:59.953Z 5 Acked 1004 Read 7 Done Done Sent false
138452024-02-20T22:46:59.953Z 6 Acked 1005 Read 7 Done Done Sent false
138462024-02-20T22:46:59.954Z 7 Acked 1006 Read 10 Done Done Sent false
138472024-02-20T22:46:59.954Z 8 Acked 1007 Write 1 Done Done Sent false
138482024-02-20T22:46:59.954Z 9 Acked 1008 Write 1 Done Done Sent false
138492024-02-20T22:46:59.954Z 10 Acked 1009 Read 8 Done Done Sent false
138502024-02-20T22:46:59.954Z 11 Acked 1010 Read 7 Done Done Sent false
138512024-02-20T22:46:59.954Z 12 Acked 1011 Write 3 Done Done Sent false
138522024-02-20T22:46:59.954Z 13 Acked 1012 Write 7 Done Done Sent false
138532024-02-20T22:46:59.954Z 14 Acked 1013 Flush 0 Done Done Sent false
138542024-02-20T22:46:59.954Z 15 Acked 1014 Write 2 Done Done Sent false
138552024-02-20T22:46:59.954Z 16 Acked 1015 Read 10 Done Done Sent false
138562024-02-20T22:46:59.954Z 17 Acked 1016 Read 5 Done Done Sent false
138572024-02-20T22:46:59.954Z 18 Acked 1017 Read 3 Done Done Sent false
138582024-02-20T22:46:59.954Z 19 Acked 1018 Read 3 Done Done Sent false
138592024-02-20T22:46:59.954Z 20 Acked 1019 Write 10 Done Done Sent false
138602024-02-20T22:46:59.954Z 21 Acked 1020 Read 1 Done Done Sent false
138612024-02-20T22:46:59.954Z 22 Acked 1021 Write 6 Done Done Sent false
138622024-02-20T22:46:59.954Z 23 Acked 1022 Read 1 Done Done Sent false
138632024-02-20T22:46:59.954Z 24 Acked 1023 Read 3 Done Done Sent false
138642024-02-20T22:46:59.954Z 25 Acked 1024 Write 10 Done Done Sent false
138652024-02-20T22:46:59.954Z 26 Acked 1025 Write 1 Done Done Sent false
138662024-02-20T22:46:59.954Z 27 Acked 1026 Read 8 Done Done Sent false
138672024-02-20T22:46:59.954Z 28 Acked 1027 Write 1 Done Done Sent false
138682024-02-20T22:46:59.954Z 29 Acked 1028 Read 2 Done Done Sent false
138692024-02-20T22:46:59.954Z 30 Acked 1029 Read 3 Done Sent Sent false
138702024-02-20T22:46:59.954Z STATES DS:0 DS:1 DS:2 TOTAL
138712024-02-20T22:46:59.955Z New 0 0 0 0
138722024-02-20T22:46:59.955Z Sent 0 1 30 31
138732024-02-20T22:46:59.955Z Done 30 29 0 59
138742024-02-20T22:46:59.955Z Skipped 0 0 0 0
138752024-02-20T22:46:59.955Z Error 0 0 0 0
138762024-02-20T22:46:59.955ZLast Flush: 1013 1013 0
138772024-02-20T22:46:59.955ZDownstairs last five completed:
138782024-02-20T22:46:59.955ZUpstairs last five completed: 30 29 28 27 26
138792024-02-20T22:46:59.955Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:46:59.467158121Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1650}
138802024-02-20T22:46:59.955ZWrote out file "/tmp/test_repair_verify.data"
138812024-02-20T22:46:59.959Z
138822024-02-20T22:46:59.960ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138832024-02-20T22:47:00.387ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138842024-02-20T22:47:00.387Z 0 000-019 126 126 126 219 219 219 F F F
138852024-02-20T22:47:00.387Z 1 020-039 122 122 122 210 210 210 T T F <---
138862024-02-20T22:47:00.387Z 2 040-059 123 123 123 212 212 212 F F F
138872024-02-20T22:47:00.387Z 3 060-079 127 127 127 224 224 224 F F F
138882024-02-20T22:47:00.387Z 4 080-099 126 126 126 222 222 222 F F F
138892024-02-20T22:47:00.387Z 5 100-119 128 128 124 225 225 214 F F F <---
138902024-02-20T22:47:00.387Z 6 120-139 122 122 122 210 210 210 F F F
138912024-02-20T22:47:00.387Z 7 140-159 126 126 126 217 217 217 T T F <---
138922024-02-20T22:47:00.397Z 8 160-179 128 128 124 225 225 215 F F F <---
138932024-02-20T22:47:00.397Z 9 180-199 124 124 124 215 215 215 F F F
138942024-02-20T22:47:00.397Z 10 200-219 126 126 126 219 219 219 F F F
138952024-02-20T22:47:00.397Z 11 220-239 126 126 126 223 223 223 F F F
138962024-02-20T22:47:00.397Z 12 240-259 126 126 126 223 223 223 F F F
138972024-02-20T22:47:00.397Z 13 260-279 128 128 127 225 225 224 F F F <---
138982024-02-20T22:47:00.397Z 14 280-299 122 122 122 211 211 211 T T F <---
138992024-02-20T22:47:00.397Z 15 300-319 123 123 123 212 212 212 T T F <---
139002024-02-20T22:47:00.397Z 16 320-339 128 128 124 225 225 214 F F F <---
139012024-02-20T22:47:00.397Z 17 340-359 128 128 125 225 225 216 F F F <---
139022024-02-20T22:47:00.398Z 18 360-379 128 128 125 225 225 216 F F F <---
139032024-02-20T22:47:00.398Z 19 380-399 126 126 126 222 222 222 F F F
139042024-02-20T22:47:00.398Z 20 400-419 122 122 122 210 210 210 F F F
139052024-02-20T22:47:00.398Z 21 420-439 112 112 112 197 197 197 T T F <---
139062024-02-20T22:47:00.398Z 22 440-459 123 123 123 212 212 212 T T F <---
139072024-02-20T22:47:00.398Z 23 460-479 126 126 126 222 222 222 F F F
139082024-02-20T22:47:00.398Z 24 480-499 126 126 126 222 222 222 F F F
139092024-02-20T22:47:00.398Z 25 500-519 124 124 124 215 215 215 F F F
139102024-02-20T22:47:00.398Z 26 520-539 123 123 123 212 212 212 F F F
139112024-02-20T22:47:00.398Z 27 540-559 128 128 117 225 225 203 T T F <---
139122024-02-20T22:47:00.398Z 28 560-579 123 123 123 212 212 212 F F F
139132024-02-20T22:47:00.398Z 29 580-599 123 123 123 212 212 212 F F F
139142024-02-20T22:47:00.398ZMax gen: 128, Max flush: 225
139152024-02-20T22:47:00.398ZError: Difference in extent metadata found!
139162024-02-20T22:47:00.398ZA Difference in extent metadata is expected here
139172024-02-20T22:47:00.398ZOn loop 64
139182024-02-20T22:47:00.398Z
139192024-02-20T22:47:00.398ZVerifying data now
139202024-02-20T22:47:04.627ZNo change after verify
139212024-02-20T22:47:04.627ZLoop: 64 Downstairs dump after verify (and repair):
139222024-02-20T22:47:04.642ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139232024-02-20T22:47:04.642Z 0 000-019 126 126 126 219 219 219 F F F
139242024-02-20T22:47:04.642Z 1 020-039 129 129 129 226 226 226 F F F
139252024-02-20T22:47:04.642Z 2 040-059 123 123 123 212 212 212 F F F
139262024-02-20T22:47:04.642Z 3 060-079 127 127 127 224 224 224 F F F
139272024-02-20T22:47:04.642Z 4 080-099 126 126 126 222 222 222 F F F
139282024-02-20T22:47:04.642Z 5 100-119 128 128 128 225 225 225 F F F
139292024-02-20T22:47:04.642Z 6 120-139 122 122 122 210 210 210 F F F
139302024-02-20T22:47:04.642Z 7 140-159 129 129 129 226 226 226 F F F
139312024-02-20T22:47:04.642Z 8 160-179 128 128 128 225 225 225 F F F
139322024-02-20T22:47:04.647Z 9 180-199 124 124 124 215 215 215 F F F
139332024-02-20T22:47:04.647Z 10 200-219 126 126 126 219 219 219 F F F
139342024-02-20T22:47:04.647Z 11 220-239 126 126 126 223 223 223 F F F
139352024-02-20T22:47:04.647Z 12 240-259 126 126 126 223 223 223 F F F
139362024-02-20T22:47:04.647Z 13 260-279 128 128 128 225 225 225 F F F
139372024-02-20T22:47:04.647Z 14 280-299 129 129 129 226 226 226 F F F
139382024-02-20T22:47:04.647Z 15 300-319 129 129 129 226 226 226 F F F
139392024-02-20T22:47:04.647Z 16 320-339 128 128 128 225 225 225 F F F
139402024-02-20T22:47:04.647Z 17 340-359 128 128 128 225 225 225 F F F
139412024-02-20T22:47:04.647Z 18 360-379 128 128 128 225 225 225 F F F
139422024-02-20T22:47:04.647Z 19 380-399 126 126 126 222 222 222 F F F
139432024-02-20T22:47:04.647Z 20 400-419 122 122 122 210 210 210 F F F
139442024-02-20T22:47:04.647Z 21 420-439 129 129 129 226 226 226 F F F
139452024-02-20T22:47:04.647Z 22 440-459 129 129 129 226 226 226 F F F
139462024-02-20T22:47:04.647Z 23 460-479 126 126 126 222 222 222 F F F
139472024-02-20T22:47:04.648Z 24 480-499 126 126 126 222 222 222 F F F
139482024-02-20T22:47:04.648Z 25 500-519 124 124 124 215 215 215 F F F
139492024-02-20T22:47:04.648Z 26 520-539 123 123 123 212 212 212 F F F
139502024-02-20T22:47:04.648Z 27 540-559 129 129 129 226 226 226 F F F
139512024-02-20T22:47:04.648Z 28 560-579 123 123 123 212 212 212 F F F
139522024-02-20T22:47:04.648Z 29 580-599 123 123 123 212 212 212 F F F
139532024-02-20T22:47:04.648ZMax gen: 129, Max flush: 226
139542024-02-20T22:47:04.648Z
139552024-02-20T22:47:04.648ZBegin loop 65
139562024-02-20T22:47:04.648ZDownstairs to restart: 2
139572024-02-20T22:47:04.663Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.178687759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658}
139582024-02-20T22:47:04.673Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
139592024-02-20T22:47:04.673Z2024-02-20T22:47:04.178995728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658}
139602024-02-20T22:47:04.673Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179135123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658}
139612024-02-20T22:47:04.673Z{"msg":"Crucible 13c599bf-e143-4039-a63d-9c2fa7910a9e has session id: 27200c2a-2542-4ad0-97b7-5bd5fb657da7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.17921758Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139622024-02-20T22:47:04.673Z{"msg":"Upstairs opts: Upstairs UUID: 13c599bf-e143-4039-a63d-9c2fa7910a9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179258938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139632024-02-20T22:47:04.673Z{"msg":"Crucible stats registered with UUID: 13c599bf-e143-4039-a63d-9c2fa7910a9e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179278707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139642024-02-20T22:47:04.673Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179315406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658}
139652024-02-20T22:47:04.674Z{"msg":"13c599bf-e143-4039-a63d-9c2fa7910a9e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179339875Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139662024-02-20T22:47:04.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179378394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139672024-02-20T22:47:04.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179411813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139682024-02-20T22:47:04.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179461941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139692024-02-20T22:47:04.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.17949203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139702024-02-20T22:47:04.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179530058Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139712024-02-20T22:47:04.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179629425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139722024-02-20T22:47:04.674Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179691603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139732024-02-20T22:47:04.674Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179774089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139742024-02-20T22:47:04.674Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179803899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139752024-02-20T22:47:04.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179849607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139762024-02-20T22:47:04.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179874756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139772024-02-20T22:47:04.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.179908105Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139782024-02-20T22:47:04.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.18031454Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139792024-02-20T22:47:04.674Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180352589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139802024-02-20T22:47:04.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180480924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139812024-02-20T22:47:04.674Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180526833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139822024-02-20T22:47:04.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180558581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139832024-02-20T22:47:04.674Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180613259Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139842024-02-20T22:47:04.674Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180916719Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139852024-02-20T22:47:04.674Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.180967327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139862024-02-20T22:47:04.674Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181219538Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139872024-02-20T22:47:04.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181474009Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139882024-02-20T22:47:04.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181512587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139892024-02-20T22:47:04.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181529247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139902024-02-20T22:47:04.675Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181555006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139912024-02-20T22:47:04.675Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181578035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139922024-02-20T22:47:04.675Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181592555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139932024-02-20T22:47:04.675Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181634833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139942024-02-20T22:47:04.675Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181676961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139952024-02-20T22:47:04.675Z{"msg":"[0]R flush_numbers[0..12]: [219, 226, 212, 224, 222, 225, 210, 226, 225, 215, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.18170514Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139962024-02-20T22:47:04.675Z{"msg":"[0]R generation[0..12]: [126, 129, 123, 127, 126, 128, 122, 129, 128, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181737889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139972024-02-20T22:47:04.675Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181753919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139982024-02-20T22:47:04.675Z{"msg":"[1]R flush_numbers[0..12]: [219, 226, 212, 224, 222, 225, 210, 226, 225, 215, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181776108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
139992024-02-20T22:47:04.675Z{"msg":"[1]R generation[0..12]: [126, 129, 123, 127, 126, 128, 122, 129, 128, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181790277Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140002024-02-20T22:47:04.675Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181815956Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140012024-02-20T22:47:04.675Z{"msg":"[2]R flush_numbers[0..12]: [219, 226, 212, 224, 222, 225, 210, 226, 225, 215, 219, 223]","v":0,"name"Wait for a query_work_queue command to finish before sending IO
140022024-02-20T22:47:04.675Z:"crucible","level":30,"time":"2024-02-20T22:47:04.181849495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140032024-02-20T22:47:04.675Z{"msg":"[2]R generation[0..12]: [126, 129, 123, 127, 126, 128, 122, 129, 128, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181870685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140042024-02-20T22:47:04.675Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181905083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140052024-02-20T22:47:04.675Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181921413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140062024-02-20T22:47:04.675Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181938122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140072024-02-20T22:47:04.675Z{"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.181971971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140082024-02-20T22:47:04.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182023389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140092024-02-20T22:47:04.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182054408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140102024-02-20T22:47:04.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182073507Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140112024-02-20T22:47:04.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182105166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140122024-02-20T22:47:04.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182125965Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140132024-02-20T22:47:04.676Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182145675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140142024-02-20T22:47:04.676Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182172484Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140152024-02-20T22:47:04.676Z{"msg":"13c599bf-e143-4039-a63d-9c2fa7910a9e is now active with session: 27200c2a-2542-4ad0-97b7-5bd5fb657da7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182186743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140162024-02-20T22:47:04.676Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182199533Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658,"session_id":"27200c2a-2542-4ad0-97b7-5bd5fb657da7"}
140172024-02-20T22:47:04.676Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:04.182212642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658}
140182024-02-20T22:47:04.676ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140192024-02-20T22:47:04.676ZLoading write count information from file "/tmp/test_repair_verify.data"
140202024-02-20T22:47:04.676ZRun Repair workload
140212024-02-20T22:47:04.676Z01/30 Write block 12 len 12288 data: 5 7 6
140222024-02-20T22:47:04.676Z02/30 Read block 485 len 16384
140232024-02-20T22:47:04.681Z03/30 Read block 262 len 40960
140242024-02-20T22:47:04.681Z04/30 Read block 440 len 4096
140252024-02-20T22:47:04.681Z05/30 Read block 159 len 20480
140262024-02-20T22:47:04.681Z06/30 Flush
140272024-02-20T22:47:04.681Z07/30 Write block 32 len 32768 data: 19 19 16 17 14 13 13 12
140282024-02-20T22:47:04.681Z08/30 Flush
140292024-02-20T22:47:04.681Z09/30 Write block 325 len 28672 data: 13 14 12 12 11 10 9
140302024-02-20T22:47:04.681Z10/30 Write block 400 len 28672 data: 12 10 12 13 12 11 11
140312024-02-20T22:47:04.681Z11/30 Write block 246 len 40960 data: 7 11 10 7 8 8 8 10 9 7
140322024-02-20T22:47:04.681Z12/30 Read block 473 len 40960
140332024-02-20T22:47:04.685Z13/30 Write block 278 len 16384 data: 4 5 6 8
140342024-02-20T22:47:04.685Z14/30 Write block 366 len 24576 data: 15 11 11 12 15 18
140352024-02-20T22:47:04.685Z15/30 Write block 370 len 12288 data: 16 19 15
140362024-02-20T22:47:04.686Z16/30 Flush
140372024-02-20T22:47:04.689Z17/30 Write block 99 len 24576 data: 10 8 8 7 5 7
140382024-02-20T22:47:04.690Z18/30 Read block 183 len 40960
140392024-02-20T22:47:04.690Z19/30 Read block 258 len 32768
140402024-02-20T22:47:04.690Z20/30 Flush
140412024-02-20T22:47:04.693Z21/30 Write block 413 len 8192 data: 17 18
140422024-02-20T22:47:04.693Z22/30 Read block 361 len 28672
140432024-02-20T22:47:04.699Z23/30 Read block 508 len 24576
140442024-02-20T22:47:04.699Z24/30 Read block 77 len 36864
140452024-02-20T22:47:04.699Z25/30 Read block 15 len 4096
140462024-02-20T22:47:04.699Z26/30 Flush
140472024-02-20T22:47:04.699Z27/30 Write block 181 len 16384 data: 15 14 11 12
140482024-02-20T22:47:04.703Z28/30 Write block 510 len 8192 data: 11 14
140492024-02-20T22:47:04.704Z29/30 Read block 262 len 24576
140502024-02-20T22:47:04.704Z30/30 Write block 316 len 12288 data: 13 15 11
140512024-02-20T22:47:04.704Z----------------------------------------------------------------
140522024-02-20T22:47:04.704Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
140532024-02-20T22:47:04.704ZStates: Active Active Active
140542024-02-20T22:47:04.704ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140552024-02-20T22:47:04.704Z 1 Acked 1000 Write 3 Done Done Done false
140562024-02-20T22:47:04.704Z 2 Acked 1001 Read 4 Done Done Done false
140572024-02-20T22:47:04.704Z 3 Acked 1002 Read 10 Done Done Done false
140582024-02-20T22:47:04.704Z 4 Acked 1003 Read 1 Done Done Done false
140592024-02-20T22:47:04.704Z 5 Acked 1004 Read 5 Done Done Done false
140602024-02-20T22:47:04.704Z 6 Acked 1005 Flush 0 Done Done Sent false
140612024-02-20T22:47:04.705Z 7 Acked 1006 Write 8 Done Done Sent false
140622024-02-20T22:47:04.705Z 8 Acked 1007 Flush 0 Done Done Sent false
140632024-02-20T22:47:04.705Z 9 Acked 1008 Write 7 Done Done Sent false
140642024-02-20T22:47:04.705Z 10 Acked 1009 Write 7 Done Done Sent false
140652024-02-20T22:47:04.705Z 11 Acked 1010 Write 10 Done Done Sent false
140662024-02-20T22:47:04.705Z 12 Acked 1011 Read 10 Done Done Sent false
140672024-02-20T22:47:04.705Z 13 Acked 1012 Write 4 Done Done Sent false
140682024-02-20T22:47:04.705Z 14 Acked 1013 Write 6 Done Done Sent false
140692024-02-20T22:47:04.705Z 15 Acked 1014 Write 3 Done Done Sent false
140702024-02-20T22:47:04.705Z 16 Acked 1015 Flush 0 Done Done Sent false
140712024-02-20T22:47:04.705Z 17 Acked 1016 Write 6 Done Done Sent false
140722024-02-20T22:47:04.705Z 18 Acked 1017 Read 10 Done Done Sent false
140732024-02-20T22:47:04.705Z 19 Acked 1018 Read 8 Done Done Sent false
140742024-02-20T22:47:04.706Z 20 Acked 1019 Flush 0 Done Done Sent false
140752024-02-20T22:47:04.706Z 21 Acked 1020 Write 2 Done Done Sent false
140762024-02-20T22:47:04.706Z 22 Acked 1021 Read 7 Done Done Sent false
140772024-02-20T22:47:04.706Z 23 Acked 1022 Read 6 Done Done Sent false
140782024-02-20T22:47:04.706Z 24 Acked 1023 Read 9 Done Done Sent false
140792024-02-20T22:47:04.706Z 25 Acked 1024 Read 1 Done Done Sent false
140802024-02-20T22:47:04.706Z 26 Acked 1025 Flush 0 Done Done Sent false
140812024-02-20T22:47:04.706Z 27 Acked 1026 Write 4 Done Done Sent false
140822024-02-20T22:47:04.706Z 28 Acked 1027 Write 2 Done Done Sent false
140832024-02-20T22:47:04.706Z 29 Acked 1028 Read 6 Done Done Sent false
140842024-02-20T22:47:04.706Z 30 Acked 1029 Write 3 Sent Sent Sent false
140852024-02-20T22:47:04.706Z STATES DS:0 DS:1 DS:2 TOTAL
140862024-02-20T22:47:04.706Z New 0 0 0 0
140872024-02-20T22:47:04.707Z Sent 1 1 25 27
140882024-02-20T22:47:04.707Z Done 29 29 5 63
140892024-02-20T22:47:04.707Z Skipped 0 0 0 0
140902024-02-20T22:47:04.707Z Error 0 0 0 0
140912024-02-20T22:47:04.707ZLast Flush: 1025 1025 0
140922024-02-20T22:47:04.707ZDownstairs last five completed:
140932024-02-20T22:47:04.707ZUpstairs last five completed: 30 29 28 27 26
140942024-02-20T22:47:04.707Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:04.217441552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1658}
140952024-02-20T22:47:04.707ZWrote out file "/tmp/test_repair_verify.data"
140962024-02-20T22:47:04.711Z
140972024-02-20T22:47:04.712ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140982024-02-20T22:47:04.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140992024-02-20T22:47:04.726Z 0 000-019 130 130 126 227 227 219 F F T <---
141002024-02-20T22:47:04.726Z 1 020-039 130 130 129 228 228 226 F F F <---
141012024-02-20T22:47:04.726Z 2 040-059 123 123 123 212 212 212 F F F
141022024-02-20T22:47:04.726Z 3 060-079 127 127 127 224 224 224 F F F
141032024-02-20T22:47:04.726Z 4 080-099 130 130 126 230 230 222 F F F <---
141042024-02-20T22:47:04.726Z 5 100-119 130 130 128 230 230 225 F F F <---
141052024-02-20T22:47:04.731Z 6 120-139 122 122 122 210 210 210 F F F
141062024-02-20T22:47:04.731Z 7 140-159 129 129 129 226 226 226 F F F
141072024-02-20T22:47:04.731Z 8 160-179 128 128 128 225 225 225 F F F
141082024-02-20T22:47:04.731Z 9 180-199 124 124 124 215 215 215 T T F <---
141092024-02-20T22:47:04.731Z 10 200-219 126 126 126 219 219 219 F F F
141102024-02-20T22:47:04.731Z 11 220-239 126 126 126 223 223 223 F F F
141112024-02-20T22:47:04.731Z 12 240-259 130 130 126 229 229 223 F F F <---
141122024-02-20T22:47:04.731Z 13 260-279 130 130 128 229 229 225 F F F <---
141132024-02-20T22:47:04.731Z 14 280-299 130 130 129 229 229 226 F F F <---
141142024-02-20T22:47:04.731Z 15 300-319 129 129 129 226 226 226 T T F <---
141152024-02-20T22:47:04.731Z 16 320-339 130 130 128 229 229 225 F F F <---
141162024-02-20T22:47:04.731Z 17 340-359 128 128 128 225 225 225 F F F
141172024-02-20T22:47:04.731Z 18 360-379 130 130 128 229 229 225 F F F <---
141182024-02-20T22:47:04.731Z 19 380-399 126 126 126 222 222 222 F F F
141192024-02-20T22:47:04.732Z 20 400-419 130 130 122 231 231 210 F F F <---
141202024-02-20T22:47:04.732Z 21 420-439 129 129 129 226 226 226 F F F
141212024-02-20T22:47:04.732Z 22 440-459 129 129 129 226 226 226 F F F
141222024-02-20T22:47:04.732Z 23 460-479 126 126 126 222 222 222 F F F
141232024-02-20T22:47:04.732Z 24 480-499 126 126 126 222 222 222 F F F
141242024-02-20T22:47:04.732Z 25 500-519 124 124 124 215 215 215 T T F <---
141252024-02-20T22:47:04.732Z 26 520-539 123 123 123 212 212 212 F F F
141262024-02-20T22:47:04.732Z 27 540-559 129 129 129 226 226 226 F F F
141272024-02-20T22:47:04.732Z 28 560-579 123 123 123 212 212 212 F F F
141282024-02-20T22:47:04.732Z 29 580-599 123 123 123 212 212 212 F F F
141292024-02-20T22:47:04.732ZMax gen: 130, Max flush: 231
141302024-02-20T22:47:04.732ZError: Difference in extent metadata found!
141312024-02-20T22:47:04.732ZA Difference in extent metadata is expected here
141322024-02-20T22:47:04.732ZOn loop 65
141332024-02-20T22:47:04.732Z
141342024-02-20T22:47:04.732ZVerifying data now
141352024-02-20T22:47:09.370ZNo change after verify
141362024-02-20T22:47:09.370ZLoop: 65 Downstairs dump after verify (and repair):
141372024-02-20T22:47:09.384ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141382024-02-20T22:47:09.384Z 0 000-019 130 130 130 227 227 227 F F F
141392024-02-20T22:47:09.384Z 1 020-039 130 130 130 228 228 228 F F F
141402024-02-20T22:47:09.384Z 2 040-059 123 123 123 212 212 212 F F F
141412024-02-20T22:47:09.384Z 3 060-079 127 127 127 224 224 224 F F F
141422024-02-20T22:47:09.385Z 4 080-099 130 130 130 230 230 230 F F F
141432024-02-20T22:47:09.385Z 5 100-119 130 130 130 230 230 230 F F F
141442024-02-20T22:47:09.385Z 6 120-139 122 122 122 210 210 210 F F F
141452024-02-20T22:47:09.385Z 7 140-159 129 129 129 226 226 226 F F F
141462024-02-20T22:47:09.385Z 8 160-179 128 128 128 225 225 225 F F F
141472024-02-20T22:47:09.385Z 9 180-199 131 131 131 232 232 232 F F F
141482024-02-20T22:47:09.385Z 10 200-219 126 126 126 219 219 219 F F F
141492024-02-20T22:47:09.385Z 11 220-239 126 126 126 223 223 223 F F F
141502024-02-20T22:47:09.385Z 12 240-259 130 130 130 229 229 229 F F F
141512024-02-20T22:47:09.385Z 13 260-279 130 130 130 229 229 229 F F F
141522024-02-20T22:47:09.385Z 14 280-299 130 130 130 229 229 229 F F F
141532024-02-20T22:47:09.385Z 15 300-319 131 131 131 232 232 232 F F F
141542024-02-20T22:47:09.391Z 16 320-339 130 130 130 229 229 229 F F F
141552024-02-20T22:47:09.391Z 17 340-359 128 128 128 225 225 225 F F F
141562024-02-20T22:47:09.391Z 18 360-379 130 130 130 229 229 229 F F F
141572024-02-20T22:47:09.391Z 19 380-399 126 126 126 222 222 222 F F F
141582024-02-20T22:47:09.391Z 20 400-419 130 130 130 231 231 231 F F F
141592024-02-20T22:47:09.391Z 21 420-439 129 129 129 226 226 226 F F F
141602024-02-20T22:47:09.391Z 22 440-459 129 129 129 226 226 226 F F F
141612024-02-20T22:47:09.391Z 23 460-479 126 126 126 222 222 222 F F F
141622024-02-20T22:47:09.391Z 24 480-499 126 126 126 222 222 222 F F F
141632024-02-20T22:47:09.391Z 25 500-519 131 131 131 232 232 232 F F F
141642024-02-20T22:47:09.392Z 26 520-539 123 123 123 212 212 212 F F F
141652024-02-20T22:47:09.392Z 27 540-559 129 129 129 226 226 226 F F F
141662024-02-20T22:47:09.392Z 28 560-579 123 123 123 212 212 212 F F F
141672024-02-20T22:47:09.392Z 29 580-599 123 123 123 212 212 212 F F F
141682024-02-20T22:47:09.392ZMax gen: 131, Max flush: 232
141692024-02-20T22:47:09.392Z
141702024-02-20T22:47:09.392ZBegin loop 66
141712024-02-20T22:47:09.392ZDownstairs to restart: 1
141722024-02-20T22:47:09.409Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924341374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666}
141732024-02-20T22:47:09.409Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924599465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666}
141742024-02-20T22:47:09.409Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924618435Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666}
141752024-02-20T22:47:09.419Z{"msg":"Crucible 59607891-698d-4531-8a8f-9a7272c31edd has session id: e4b211f8-5f5a-4577-be85-3005f5a0c027","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924633284Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141762024-02-20T22:47:09.419Z{"msg":"Upstairs opts: Upstairs UUID: 59607891-698d-4531-8a8f-9a7272c31edd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924677722Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141772024-02-20T22:47:09.419ZCrucible runtime is spawned
141782024-02-20T22:47:09.419Z{"msg":"Crucible stats registered with UUID: 59607891-698d-4531-8a8f-9a7272c31edd","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924714741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141792024-02-20T22:47:09.419Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.924825937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666}
141802024-02-20T22:47:09.419Z{"msg":"59607891-698d-4531-8a8f-9a7272c31edd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.92503958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141812024-02-20T22:47:09.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925098058Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141822024-02-20T22:47:09.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925162635Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141832024-02-20T22:47:09.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925206014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141842024-02-20T22:47:09.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925243152Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141852024-02-20T22:47:09.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.9252967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141862024-02-20T22:47:09.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925337819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141872024-02-20T22:47:09.419Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925372678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141882024-02-20T22:47:09.419Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925410796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141892024-02-20T22:47:09.419Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925444695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141902024-02-20T22:47:09.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925495593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141912024-02-20T22:47:09.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925533422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141922024-02-20T22:47:09.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.925608949Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141932024-02-20T22:47:09.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926167619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141942024-02-20T22:47:09.419Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926211268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141952024-02-20T22:47:09.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926229427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141962024-02-20T22:47:09.420Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926244117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141972024-02-20T22:47:09.420Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926267036Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141982024-02-20T22:47:09.420Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926280925Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
141992024-02-20T22:47:09.420Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926833835Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142002024-02-20T22:47:09.420Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926879774Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142012024-02-20T22:47:09.420Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.926914073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142022024-02-20T22:47:09.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927392415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142032024-02-20T22:47:09.420Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927436254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142042024-02-20T22:47:09.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927454003Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142052024-02-20T22:47:09.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927475402Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142062024-02-20T22:47:09.420Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927490232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142072024-02-20T22:47:09.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927511931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142082024-02-20T22:47:09.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927524981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142092024-02-20T22:47:09.420Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.92754553Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142102024-02-20T22:47:09.420Z{"msg":"[0]R flush_numbers[0..12]: [227, 228, 212, 224, 230, 230, 210, 226, 225, 232, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927560349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142112024-02-20T22:47:09.420Z{"msg":"[0]R generation[0..12]: [130, 130, 123, 127, 130, 130, 122, 129, 128, 131, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927581229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142122024-02-20T22:47:09.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927595728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142132024-02-20T22:47:09.420Z{"msg":"[1]R flush_numbers[0..12]: [227, 228, 212, 224, 230, 230, 210, 226, 225, 232, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927610057Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142142024-02-20T22:47:09.420Z{"msg":"[1]R generation[0..12]: [130, 130, 123, 127, 130, 130, 122, 129, 128, 131, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927630797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142152024-02-20T22:47:09.420Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927645456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142162024-02-20T22:47:09.420ZWait for a query_work_queue command to finish before sending IO
142172024-02-20T22:47:09.420Z{"msg":"[2]R flush_numbers[0..12]: [227, 228, 212, 224, 230, 230, 210, 226, 225, 232, 219, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927664216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142182024-02-20T22:47:09.420Z{"msg":"[2]R generation[0..12]: [130, 130, 123, 127, 130, 130, 122, 129, 128, 131, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927678425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142192024-02-20T22:47:09.421Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927699624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142202024-02-20T22:47:09.421Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927713664Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142212024-02-20T22:47:09.421Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927735453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142222024-02-20T22:47:09.421Z{"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927748972Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142232024-02-20T22:47:09.421Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927769522Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142242024-02-20T22:47:09.421Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927783221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142252024-02-20T22:47:09.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927795541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142262024-02-20T22:47:09.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.92781741Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142272024-02-20T22:47:09.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.92783149Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142282024-02-20T22:47:09.421Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927864788Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142292024-02-20T22:47:09.421Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927884828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142302024-02-20T22:47:09.421Z{"msg":"59607891-698d-4531-8a8f-9a7272c31edd is now active with session: e4b211f8-5f5a-4577-be85-3005f5a0c027","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927907037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142312024-02-20T22:47:09.421Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927919817Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666,"session_id":"e4b211f8-5f5a-4577-be85-3005f5a0c027"}
142322024-02-20T22:47:09.421Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:08.927939496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666}
142332024-02-20T22:47:09.421ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142342024-02-20T22:47:09.421ZLoading write count information from file "/tmp/test_repair_verify.data"
142352024-02-20T22:47:09.421ZRun Repair workload
142362024-02-20T22:47:09.421Z01/30 Write block 300 len 4096 data: 11
142372024-02-20T22:47:09.427Z02/30 Write block 206 len 24576 data: 11 11 9 9 10 10
142382024-02-20T22:47:09.427Z03/30 Write block 1 len 20480 data: 3 8 9 11 11
142392024-02-20T22:47:09.427Z04/30 Read block 33 len 12288
142402024-02-20T22:47:09.427Z05/30 Write block 139 len 16384 data: 8 9 9 10
142412024-02-20T22:47:09.427Z06/30 Read block 381 len 20480
142422024-02-20T22:47:09.427Z07/30 Read block 53 len 36864
142432024-02-20T22:47:09.427Z08/30 Read block 219 len 40960
142442024-02-20T22:47:09.427Z09/30 Write block 226 len 24576 data: 7 8 11 15 19 17
142452024-02-20T22:47:09.427Z10/30 Flush
142462024-02-20T22:47:09.427Z11/30 Write block 100 len 40960 data: 9 9 8 6 8 7 7 7 10 10
142472024-02-20T22:47:09.427Z12/30 Flush
142482024-02-20T22:47:09.431Z13/30 Write block 305 len 16384 data: 14 10 12 11
142492024-02-20T22:47:09.431Z14/30 Read block 424 len 20480
142502024-02-20T22:47:09.431Z15/30 Write block 39 len 4096 data: 13
142512024-02-20T22:47:09.431Z16/30 Read block 330 len 32768
142522024-02-20T22:47:09.439Z17/30 Read block 326 len 24576
142532024-02-20T22:47:09.439Z18/30 Read block 311 len 32768
142542024-02-20T22:47:09.439Z19/30 Read block 62 len 20480
142552024-02-20T22:47:09.439Z20/30 Flush
142562024-02-20T22:47:09.447Z21/30 Write block 516 len 36864 data: 8 9 7 6 10 11 10 8 7
142572024-02-20T22:47:09.447Z22/30 Write block 33 len 20480 data: 20 17 18 15 14
142582024-02-20T22:47:09.447Z23/30 Write block 228 len 16384 data: 12 16 20 18
142592024-02-20T22:47:09.447Z24/30 Flush
142602024-02-20T22:47:09.447Z25/30 Write block 224 len 16384 data: 7 6 8 9
142612024-02-20T22:47:09.447Z26/30 Read block 428 len 40960
142622024-02-20T22:47:09.447Z27/30 Write block 424 len 32768 data: 7 9 8 6 8 8 7 6
142632024-02-20T22:47:09.447Z28/30 Read block 38 len 28672
142642024-02-20T22:47:09.452Z29/30 Write block 332 len 40960 data: 14 16 14 12 12 13 15 14 13 13
142652024-02-20T22:47:09.452Z30/30 Write block 204 len 28672 data: 9 10 12 12 10 10 11
142662024-02-20T22:47:09.452Z----------------------------------------------------------------
142672024-02-20T22:47:09.452Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142682024-02-20T22:47:09.452ZStates: Active Active Active
142692024-02-20T22:47:09.452ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142702024-02-20T22:47:09.452Z 1 Acked 1000 Write 1 Done Sent Done false
142712024-02-20T22:47:09.452Z 2 Acked 1001 Write 6 Done Sent Done false
142722024-02-20T22:47:09.452Z 3 Acked 1002 Write 5 Done Sent Done false
142732024-02-20T22:47:09.452Z 4 Acked 1003 Read 3 Done Sent Done false
142742024-02-20T22:47:09.452Z 5 Acked 1004 Write 4 Done Sent Done false
142752024-02-20T22:47:09.452Z 6 Acked 1005 Read 5 Done Sent Done false
142762024-02-20T22:47:09.452Z 7 Acked 1006 Read 9 Done Sent Done false
142772024-02-20T22:47:09.452Z 8 Acked 1007 Read 10 Done Sent Done false
142782024-02-20T22:47:09.452Z 9 Acked 1008 Write 6 Done Sent Done false
142792024-02-20T22:47:09.452Z 10 Acked 1009 Flush 0 Done Sent Done false
142802024-02-20T22:47:09.452Z 11 Acked 1010 Write 10 Done Sent Done false
142812024-02-20T22:47:09.452Z 12 Acked 1011 Flush 0 Done Sent Done false
142822024-02-20T22:47:09.452Z 13 Acked 1012 Write 4 Done Sent Done false
142832024-02-20T22:47:09.453Z 14 Acked 1013 Read 5 Done Sent Done false
142842024-02-20T22:47:09.453Z 15 Acked 1014 Write 1 Done Sent Done false
142852024-02-20T22:47:09.453Z 16 Acked 1015 Read 8 Done Sent Done false
142862024-02-20T22:47:09.453Z 17 Acked 1016 Read 6 Done Sent Done false
142872024-02-20T22:47:09.453Z 18 Acked 1017 Read 8 Done Sent Done false
142882024-02-20T22:47:09.453Z 19 Acked 1018 Read 5 Done Sent Done false
142892024-02-20T22:47:09.453Z 20 Acked 1019 Flush 0 Done Sent Done false
142902024-02-20T22:47:09.453Z 21 Acked 1020 Write 9 Done Sent Done false
142912024-02-20T22:47:09.453Z 22 Acked 1021 Write 5 Done Sent Done false
142922024-02-20T22:47:09.453Z 23 Acked 1022 Write 4 Done Sent Done false
142932024-02-20T22:47:09.453Z 24 Acked 1023 Flush 0 Done Sent Done false
142942024-02-20T22:47:09.453Z 25 Acked 1024 Write 4 Done Sent Done false
142952024-02-20T22:47:09.453Z 26 Acked 1025 Read 10 Done Sent Done false
142962024-02-20T22:47:09.453Z 27 Acked 1026 Write 8 Done Sent Done false
142972024-02-20T22:47:09.453Z 28 Acked 1027 Read 7 Done Sent Done false
142982024-02-20T22:47:09.453Z 29 Acked 1028 Write 10 Sent Sent Sent false
142992024-02-20T22:47:09.453Z 30 Acked 1029 Write 7 Sent Sent Sent false
143002024-02-20T22:47:09.453Z STATES DS:0 DS:1 DS:2 TOTAL
143012024-02-20T22:47:09.453Z New 0 0 0 0
143022024-02-20T22:47:09.453Z Sent 2 30 2 34
143032024-02-20T22:47:09.453Z Done 28 0 28 56
143042024-02-20T22:47:09.453Z Skipped 0 0 0 0
143052024-02-20T22:47:09.453Z Error 0 0 0 0
143062024-02-20T22:47:09.453ZLast Flush: 1023 0 1023
143072024-02-20T22:47:09.454ZDownstairs last five completed:
143082024-02-20T22:47:09.454ZUpstairs last five completed: 30 29 28 27 26
143092024-02-20T22:47:09.454Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:08.96331398Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1666}
143102024-02-20T22:47:09.454ZWrote out file "/tmp/test_repair_verify.data"
143112024-02-20T22:47:09.464Z
143122024-02-20T22:47:09.464ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143132024-02-20T22:47:09.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143142024-02-20T22:47:09.472Z 0 000-019 132 130 132 233 227 233 F F F <---
143152024-02-20T22:47:09.478Z 1 020-039 132 130 132 236 228 236 F F F <---
143162024-02-20T22:47:09.478Z 2 040-059 123 123 123 212 212 212 F F F
143172024-02-20T22:47:09.478Z 3 060-079 127 127 127 224 224 224 F F F
143182024-02-20T22:47:09.478Z 4 080-099 130 130 130 230 230 230 F F F
143192024-02-20T22:47:09.478Z 5 100-119 132 130 132 234 230 234 F F F <---
143202024-02-20T22:47:09.478Z 6 120-139 132 122 132 233 210 233 F F F <---
143212024-02-20T22:47:09.479Z 7 140-159 132 129 132 233 226 233 F F F <---
143222024-02-20T22:47:09.479Z 8 160-179 128 128 128 225 225 225 F F F
143232024-02-20T22:47:09.479Z 9 180-199 131 131 131 232 232 232 F F F
143242024-02-20T22:47:09.479Z 10 200-219 132 126 132 233 219 233 T F T <---
143252024-02-20T22:47:09.479Z 11 220-239 132 126 132 236 223 236 T F T <---
143262024-02-20T22:47:09.479Z 12 240-259 130 130 130 229 229 229 F F F
143272024-02-20T22:47:09.479Z 13 260-279 130 130 130 229 229 229 F F F
143282024-02-20T22:47:09.479Z 14 280-299 130 130 130 229 229 229 F F F
143292024-02-20T22:47:09.479Z 15 300-319 132 131 132 235 232 235 F F F <---
143302024-02-20T22:47:09.479Z 16 320-339 130 130 130 229 229 229 T F T <---
143312024-02-20T22:47:09.479Z 17 340-359 128 128 128 225 225 225 T F T <---
143322024-02-20T22:47:09.479Z 18 360-379 130 130 130 229 229 229 F F F
143332024-02-20T22:47:09.479Z 19 380-399 126 126 126 222 222 222 F F F
143342024-02-20T22:47:09.479Z 20 400-419 130 130 130 231 231 231 F F F
143352024-02-20T22:47:09.479Z 21 420-439 129 129 129 226 226 226 T F T <---
143362024-02-20T22:47:09.479Z 22 440-459 129 129 129 226 226 226 F F F
143372024-02-20T22:47:09.479Z 23 460-479 126 126 126 222 222 222 F F F
143382024-02-20T22:47:09.479Z 24 480-499 126 126 126 222 222 222 F F F
143392024-02-20T22:47:09.479Z 25 500-519 132 131 132 236 232 236 F F F <---
143402024-02-20T22:47:09.479Z 26 520-539 132 123 132 236 212 236 F F F <---
143412024-02-20T22:47:09.479Z 27 540-559 129 129 129 226 226 226 F F F
143422024-02-20T22:47:09.479Z 28 560-579 123 123 123 212 212 212 F F F
143432024-02-20T22:47:09.479Z 29 580-599 123 123 123 212 212 212 F F F
143442024-02-20T22:47:09.479ZMax gen: 132, Max flush: 236
143452024-02-20T22:47:09.479ZError: Difference in extent metadata found!
143462024-02-20T22:47:09.480ZA Difference in extent metadata is expected here
143472024-02-20T22:47:09.480ZOn loop 66
143482024-02-20T22:47:09.480Z
143492024-02-20T22:47:09.485ZVerifying data now
143502024-02-20T22:47:14.124ZNo change after verify
143512024-02-20T22:47:14.124ZLoop: 66 Downstairs dump after verify (and repair):
143522024-02-20T22:47:14.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143532024-02-20T22:47:14.138Z 0 000-019 132 132 132 233 233 233 F F F
143542024-02-20T22:47:14.138Z 1 020-039 132 132 132 236 236 236 F F F
143552024-02-20T22:47:14.138Z 2 040-059 123 123 123 212 212 212 F F F
143562024-02-20T22:47:14.138Z 3 060-079 127 127 127 224 224 224 F F F
143572024-02-20T22:47:14.138Z 4 080-099 130 130 130 230 230 230 F F F
143582024-02-20T22:47:14.138Z 5 100-119 132 132 132 234 234 234 F F F
143592024-02-20T22:47:14.138Z 6 120-139 132 132 132 233 233 233 F F F
143602024-02-20T22:47:14.139Z 7 140-159 132 132 132 233 233 233 F F F
143612024-02-20T22:47:14.139Z 8 160-179 128 128 128 225 225 225 F F F
143622024-02-20T22:47:14.143Z 9 180-199 131 131 131 232 232 232 F F F
143632024-02-20T22:47:14.143Z 10 200-219 133 133 133 237 237 237 F F F
143642024-02-20T22:47:14.143Z 11 220-239 133 133 133 237 237 237 F F F
143652024-02-20T22:47:14.143Z 12 240-259 130 130 130 229 229 229 F F F
143662024-02-20T22:47:14.143Z 13 260-279 130 130 130 229 229 229 F F F
143672024-02-20T22:47:14.143Z 14 280-299 130 130 130 229 229 229 F F F
143682024-02-20T22:47:14.143Z 15 300-319 132 132 132 235 235 235 F F F
143692024-02-20T22:47:14.143Z 16 320-339 133 133 133 237 237 237 F F F
143702024-02-20T22:47:14.143Z 17 340-359 133 133 133 237 237 237 F F F
143712024-02-20T22:47:14.143Z 18 360-379 130 130 130 229 229 229 F F F
143722024-02-20T22:47:14.144Z 19 380-399 126 126 126 222 222 222 F F F
143732024-02-20T22:47:14.144Z 20 400-419 130 130 130 231 231 231 F F F
143742024-02-20T22:47:14.144Z 21 420-439 133 133 133 237 237 237 F F F
143752024-02-20T22:47:14.144Z 22 440-459 129 129 129 226 226 226 F F F
143762024-02-20T22:47:14.144Z 23 460-479 126 126 126 222 222 222 F F F
143772024-02-20T22:47:14.144Z 24 480-499 126 126 126 222 222 222 F F F
143782024-02-20T22:47:14.144Z 25 500-519 132 132 132 236 236 236 F F F
143792024-02-20T22:47:14.144Z 26 520-539 132 132 132 236 236 236 F F F
143802024-02-20T22:47:14.144Z 27 540-559 129 129 129 226 226 226 F F F
143812024-02-20T22:47:14.144Z 28 560-579 123 123 123 212 212 212 F F F
143822024-02-20T22:47:14.144Z 29 580-599 123 123 123 212 212 212 F F F
143832024-02-20T22:47:14.144ZMax gen: 133, Max flush: 237
143842024-02-20T22:47:14.144Z
143852024-02-20T22:47:14.144ZBegin loop 67
143862024-02-20T22:47:14.144ZDownstairs to restart: 0
143872024-02-20T22:47:14.161Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.675858431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674}
143882024-02-20T22:47:14.169Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.67617548Z","hostname":"Crucible runtime is spawned
143892024-02-20T22:47:14.169Zip-10-150-1-191.us-west-2.compute.internal","pid":1674}
143902024-02-20T22:47:14.169Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676290746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674}
143912024-02-20T22:47:14.169Z{"msg":"Crucible b218362c-7422-4c9c-b149-f5cd800c07ad has session id: 953f1c03-3e0b-4147-a262-c3f25640eea0","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676332494Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143922024-02-20T22:47:14.169Z{"msg":"Upstairs opts: Upstairs UUID: b218362c-7422-4c9c-b149-f5cd800c07ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676385482Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143932024-02-20T22:47:14.169Z{"msg":"Crucible stats registered with UUID: b218362c-7422-4c9c-b149-f5cd800c07ad","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676415001Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143942024-02-20T22:47:14.169Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676479419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674}
143952024-02-20T22:47:14.169Z{"msg":"b218362c-7422-4c9c-b149-f5cd800c07ad active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676501528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143962024-02-20T22:47:14.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676531777Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143972024-02-20T22:47:14.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676574355Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143982024-02-20T22:47:14.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676598035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
143992024-02-20T22:47:14.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676623164Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144002024-02-20T22:47:14.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676643823Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144012024-02-20T22:47:14.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676668222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144022024-02-20T22:47:14.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676692041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144032024-02-20T22:47:14.170Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676738599Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144042024-02-20T22:47:14.170Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676777908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144052024-02-20T22:47:14.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676818557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144062024-02-20T22:47:14.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676844676Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144072024-02-20T22:47:14.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.676866035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144082024-02-20T22:47:14.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.677542231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144092024-02-20T22:47:14.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.677598299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144102024-02-20T22:47:14.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.677628338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144112024-02-20T22:47:14.170Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.677670316Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144122024-02-20T22:47:14.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.677721474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144132024-02-20T22:47:14.170Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.677767083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144142024-02-20T22:47:14.170Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678199167Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144152024-02-20T22:47:14.170Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678236396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144162024-02-20T22:47:14.170Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.67840616Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144172024-02-20T22:47:14.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678766767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144182024-02-20T22:47:14.170Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678814815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144192024-02-20T22:47:14.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678841624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144202024-02-20T22:47:14.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678870503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144212024-02-20T22:47:14.170Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678886413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144222024-02-20T22:47:14.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678921281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144232024-02-20T22:47:14.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678943551Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144242024-02-20T22:47:14.170Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678973959Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144252024-02-20T22:47:14.170Z{"msg":"[0]R flush_numbers[0..12]: [233, 236, 212, 224, 230, 234, 233, 233, 225, 232, 237, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.678999699Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144262024-02-20T22:47:14.170Z{"msg":"[0]R generation[0..12]: [132, 132, 123, 127, 130, 132, 132, 132, 128, 131, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679028508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144272024-02-20T22:47:14.171Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679054617Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144282024-02-20T22:47:14.171Z{"msg":"[1]R flush_numbers[0..12]: [233, 236, 212, 224, 230, 234, 233, 233, 225, 232, 237, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679080976Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144292024-02-20T22:47:14.171ZWait for a query_work_queue command to finish before sending IO
144302024-02-20T22:47:14.171Z{"msg":"[1]R generation[0..12]: [132, 132, 123, 127, 130, 132, 132, 132, 128, 131, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679117404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144312024-02-20T22:47:14.171Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679152843Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144322024-02-20T22:47:14.171Z{"msg":"[2]R flush_numbers[0..12]: [233, 236, 212, 224, 230, 234, 233, 233, 225, 232, 237, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679179382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144332024-02-20T22:47:14.171Z{"msg":"[2]R generation[0..12]: [132, 132, 123, 127, 130, 132, 132, 132, 128, 131, 133, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679211461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144342024-02-20T22:47:14.171Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.67923832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144352024-02-20T22:47:14.171Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679266819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144362024-02-20T22:47:14.171Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679291138Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144372024-02-20T22:47:14.171Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679318487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144382024-02-20T22:47:14.171Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679341916Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144392024-02-20T22:47:14.171Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679374575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144402024-02-20T22:47:14.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679397904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144412024-02-20T22:47:14.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679424903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144422024-02-20T22:47:14.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679448322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144432024-02-20T22:47:14.171Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679475701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144442024-02-20T22:47:14.171Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679498121Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144452024-02-20T22:47:14.171Z{"msg":"b218362c-7422-4c9c-b149-f5cd800c07ad is now active with session: 953f1c03-3e0b-4147-a262-c3f25640eea0","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.67952712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144462024-02-20T22:47:14.171Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679549239Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674,"session_id":"953f1c03-3e0b-4147-a262-c3f25640eea0"}
144472024-02-20T22:47:14.171Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:13.679575088Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674}
144482024-02-20T22:47:14.171ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144492024-02-20T22:47:14.177ZLoading write count information from file "/tmp/test_repair_verify.data"
144502024-02-20T22:47:14.177ZRun Repair workload
144512024-02-20T22:47:14.177Z01/30 Write block 210 len 12288 data: 12 11 9
144522024-02-20T22:47:14.177Z02/30 Read block 125 len 20480
144532024-02-20T22:47:14.177Z03/30 Read block 285 len 8192
144542024-02-20T22:47:14.177Z04/30 Write block 175 len 8192 data: 12 11
144552024-02-20T22:47:14.177Z05/30 Write block 154 len 40960 data: 9 8 9 11 10 11 11 8 7 6
144562024-02-20T22:47:14.177Z06/30 Write block 517 len 28672 data: 10 8 7 11 12 11 9
144572024-02-20T22:47:14.177Z07/30 Read block 359 len 8192
144582024-02-20T22:47:14.177Z08/30 Read block 36 len 32768
144592024-02-20T22:47:14.177Z09/30 Read block 133 len 20480
144602024-02-20T22:47:14.177Z10/30 Flush
144612024-02-20T22:47:14.177Z11/30 Write block 152 len 24576 data: 14 9 10 9 10 12
144622024-02-20T22:47:14.177Z12/30 Read block 132 len 20480
144632024-02-20T22:47:14.185Z13/30 Write block 254 len 32768 data: 10 8 6 7 9 9 8 8
144642024-02-20T22:47:14.185Z14/30 Write block 48 len 16384 data: 9 11 9 9
144652024-02-20T22:47:14.185Z15/30 Read block 323 len 36864
144662024-02-20T22:47:14.185Z16/30 Flush
144672024-02-20T22:47:14.191Z17/30 Write block 577 len 12288 data: 12 15 17
144682024-02-20T22:47:14.191Z18/30 Read block 40 len 24576
144692024-02-20T22:47:14.191Z19/30 Read block 584 len 24576
144702024-02-20T22:47:14.191Z20/30 Read block 168 len 32768
144712024-02-20T22:47:14.191Z21/30 Write block 34 len 32768 data: 18 19 16 15 14 14 10 8
144722024-02-20T22:47:14.191Z22/30 Read block 300 len 32768
144732024-02-20T22:47:14.191Z23/30 Read block 54 len 36864
144742024-02-20T22:47:14.627Z24/30 Write block 142 len 12288 data: 11 12 10
144752024-02-20T22:47:14.627Z25/30 Write block 9 len 24576 data: 6 9 8 6 8 7
144762024-02-20T22:47:14.627Z26/30 Write block 90 len 24576 data: 9 9 6 6 9 8
144772024-02-20T22:47:14.627Z27/30 Write block 316 len 24576 data: 14 16 12 12 14 12
144782024-02-20T22:47:14.627Z28/30 Write block 221 len 12288 data: 7 8 8
144792024-02-20T22:47:14.627Z29/30 Write block 445 len 28672 data: 9 9 9 8 8 8 9
144802024-02-20T22:47:14.627Z30/30 Read block 247 len 12288
144812024-02-20T22:47:14.627Z----------------------------------------------------------------
144822024-02-20T22:47:14.627Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144832024-02-20T22:47:14.627ZStates: Active Active Active
144842024-02-20T22:47:14.627ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144852024-02-20T22:47:14.627Z 1 Acked 1000 Write 3 Done Done Done false
144862024-02-20T22:47:14.628Z 2 Acked 1001 Read 5 Done Done Done false
144872024-02-20T22:47:14.628Z 3 Acked 1002 Read 2 Done Done Done false
144882024-02-20T22:47:14.628Z 4 Acked 1003 Write 2 Done Done Done false
144892024-02-20T22:47:14.628Z 5 Acked 1004 Write 10 Done Done Done false
144902024-02-20T22:47:14.628Z 6 Acked 1005 Write 7 Done Done Done false
144912024-02-20T22:47:14.628Z 7 Acked 1006 Read 2 Done Done Done false
144922024-02-20T22:47:14.628Z 8 Acked 1007 Read 8 Sent Done Done false
144932024-02-20T22:47:14.628Z 9 Acked 1008 Read 5 Sent Done Done false
144942024-02-20T22:47:14.628Z 10 Acked 1009 Flush 0 Sent Done Done false
144952024-02-20T22:47:14.628Z 11 Acked 1010 Write 6 Sent Done Done false
144962024-02-20T22:47:14.628Z 12 Acked 1011 Read 5 Sent Done Done false
144972024-02-20T22:47:14.628Z 13 Acked 1012 Write 8 Sent Done Done false
144982024-02-20T22:47:14.628Z 14 Acked 1013 Write 4 Sent Done Done false
144992024-02-20T22:47:14.628Z 15 Acked 1014 Read 9 Sent Done Done false
145002024-02-20T22:47:14.628Z 16 Acked 1015 Flush 0 Sent Done Done false
145012024-02-20T22:47:14.628Z 17 Acked 1016 Write 3 Sent Done Done false
145022024-02-20T22:47:14.628Z 18 Acked 1017 Read 6 Sent Done Done false
145032024-02-20T22:47:14.628Z 19 Acked 1018 Read 6 Sent Done Done false
145042024-02-20T22:47:14.628Z 20 Acked 1019 Read 8 Sent Done Done false
145052024-02-20T22:47:14.628Z 21 Acked 1020 Write 8 Sent Done Done false
145062024-02-20T22:47:14.628Z 22 Acked 1021 Read 8 Sent Done Done false
145072024-02-20T22:47:14.628Z 23 Acked 1022 Read 9 Sent Done Done false
145082024-02-20T22:47:14.628Z 24 Acked 1023 Write 3 Sent Done Done false
145092024-02-20T22:47:14.628Z 25 Acked 1024 Write 6 Sent Done Done false
145102024-02-20T22:47:14.629Z 26 Acked 1025 Write 6 Sent Done Done false
145112024-02-20T22:47:14.629Z 27 Acked 1026 Write 6 Sent Done Done false
145122024-02-20T22:47:14.629Z 28 Acked 1027 Write 3 Sent Done Done false
145132024-02-20T22:47:14.629Z 29 Acked 1028 Write 7 Sent Done Done false
145142024-02-20T22:47:14.629Z 30 Acked 1029 Read 3 Sent Done Sent false
145152024-02-20T22:47:14.629Z STATES DS:0 DS:1 DS:2 TOTAL
145162024-02-20T22:47:14.629Z New 0 0 0 0
145172024-02-20T22:47:14.629Z Sent 23 0 1 24
145182024-02-20T22:47:14.629Z Done 7 30 29 66
145192024-02-20T22:47:14.629Z Skipped 0 0 0 0
145202024-02-20T22:47:14.629Z Error 0 0 0 0
145212024-02-20T22:47:14.629ZLast Flush: 0 1015 1015
145222024-02-20T22:47:14.629ZDownstairs last five completed:
145232024-02-20T22:47:14.629ZUpstairs last five completed: 30 29 28 27 26
145242024-02-20T22:47:14.629Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:13.710798931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1674}
145252024-02-20T22:47:14.629ZWrote out file "/tmp/test_repair_verify.data"
145262024-02-20T22:47:14.650Z
145272024-02-20T22:47:14.650ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145282024-02-20T22:47:14.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145292024-02-20T22:47:14.650Z 0 000-019 132 132 132 233 233 233 F T T <---
145302024-02-20T22:47:14.650Z 1 020-039 132 132 132 236 236 236 F T T <---
145312024-02-20T22:47:14.650Z 2 040-059 123 134 134 212 239 239 F T T <---
145322024-02-20T22:47:14.650Z 3 060-079 127 127 127 224 224 224 F F F
145332024-02-20T22:47:14.650Z 4 080-099 130 130 130 230 230 230 F T T <---
145342024-02-20T22:47:14.650Z 5 100-119 132 132 132 234 234 234 F F F
145352024-02-20T22:47:14.650Z 6 120-139 132 132 132 233 233 233 F F F
145362024-02-20T22:47:14.651Z 7 140-159 132 134 134 233 239 239 T T T <---
145372024-02-20T22:47:14.651Z 8 160-179 128 134 134 225 238 238 T F F <---
145382024-02-20T22:47:14.651Z 9 180-199 131 131 131 232 232 232 F F F
145392024-02-20T22:47:14.651Z 10 200-219 133 134 134 237 238 238 T F F <---
145402024-02-20T22:47:14.651Z 11 220-239 133 133 133 237 237 237 F T T <---
145412024-02-20T22:47:14.651Z 12 240-259 130 134 134 229 239 239 F F F <---
145422024-02-20T22:47:14.651Z 13 260-279 130 134 134 229 239 239 F F F <---
145432024-02-20T22:47:14.651Z 14 280-299 130 130 130 229 229 229 F F F
145442024-02-20T22:47:14.651Z 15 300-319 132 132 132 235 235 235 F T T <---
145452024-02-20T22:47:14.651Z 16 320-339 133 133 133 237 237 237 F T T <---
145462024-02-20T22:47:14.651Z 17 340-359 133 133 133 237 237 237 F F F
145472024-02-20T22:47:14.651Z 18 360-379 130 130 130 229 229 229 F F F
145482024-02-20T22:47:14.651Z 19 380-399 126 126 126 222 222 222 F F F
145492024-02-20T22:47:14.651Z 20 400-419 130 130 130 231 231 231 F F F
145502024-02-20T22:47:14.651Z 21 420-439 133 133 133 237 237 237 F F F
145512024-02-20T22:47:14.651Z 22 440-459 129 129 129 226 226 226 F T T <---
145522024-02-20T22:47:14.651Z 23 460-479 126 126 126 222 222 222 F F F
145532024-02-20T22:47:14.651Z 24 480-499 126 126 126 222 222 222 F F F
145542024-02-20T22:47:14.651Z 25 500-519 132 134 134 236 238 238 T F F <---
145552024-02-20T22:47:14.651Z 26 520-539 132 134 134 236 238 238 T F F <---
145562024-02-20T22:47:14.651Z 27 540-559 129 129 129 226 226 226 F F F
145572024-02-20T22:47:14.651Z 28 560-579 123 123 123 212 212 212 F T T <---
145582024-02-20T22:47:14.651Z 29 580-599 123 123 123 212 212 212 F F F
145592024-02-20T22:47:14.651ZMax gen: 134, Max flush: 239
145602024-02-20T22:47:14.651ZError: Difference in extent metadata found!
145612024-02-20T22:47:14.652ZA Difference in extent metadata is expected here
145622024-02-20T22:47:14.652ZOn loop 67
145632024-02-20T22:47:14.652Z
145642024-02-20T22:47:14.652ZVerifying data now
145652024-02-20T22:47:22.087ZNo change after verify
145662024-02-20T22:47:22.087ZLoop: 67 Downstairs dump after verify (and repair):
145672024-02-20T22:47:22.109ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145682024-02-20T22:47:22.109Z 0 000-019 135 135 135 240 240 240 F F F
145692024-02-20T22:47:22.109Z 1 020-039 135 135 135 240 240 240 F F F
145702024-02-20T22:47:22.109Z 2 040-059 135 135 135 240 240 240 F F F
145712024-02-20T22:47:22.109Z 3 060-079 127 127 127 224 224 224 F F F
145722024-02-20T22:47:22.109Z 4 080-099 135 135 135 240 240 240 F F F
145732024-02-20T22:47:22.109Z 5 100-119 132 132 132 234 234 234 F F F
145742024-02-20T22:47:22.109Z 6 120-139 132 132 132 233 233 233 F F F
145752024-02-20T22:47:22.109Z 7 140-159 135 135 135 240 240 240 F F F
145762024-02-20T22:47:22.109Z 8 160-179 134 134 134 238 238 238 F F F
145772024-02-20T22:47:22.109Z 9 180-199 131 131 131 232 232 232 F F F
145782024-02-20T22:47:22.110Z 10 200-219 134 134 134 238 238 238 F F F
145792024-02-20T22:47:22.110Z 11 220-239 135 135 135 240 240 240 F F F
145802024-02-20T22:47:22.110Z 12 240-259 134 134 134 239 239 239 F F F
145812024-02-20T22:47:22.110Z 13 260-279 134 134 134 239 239 239 F F F
145822024-02-20T22:47:22.110Z 14 280-299 130 130 130 229 229 229 F F F
145832024-02-20T22:47:22.110Z 15 300-319 135 135 135 240 240 240 F F F
145842024-02-20T22:47:22.110Z 16 320-339 135 135 135 240 240 240 F F F
145852024-02-20T22:47:22.110Z 17 340-359 133 133 133 237 237 237 F F F
145862024-02-20T22:47:22.110Z 18 360-379 130 130 130 229 229 229 F F F
145872024-02-20T22:47:22.110Z 19 380-399 126 126 126 222 222 222 F F F
145882024-02-20T22:47:22.110Z 20 400-419 130 130 130 231 231 231 F F F
145892024-02-20T22:47:22.110Z 21 420-439 133 133 133 237 237 237 F F F
145902024-02-20T22:47:22.110Z 22 440-459 135 135 135 240 240 240 F F F
145912024-02-20T22:47:22.110Z 23 460-479 126 126 126 222 222 222 F F F
145922024-02-20T22:47:22.110Z 24 480-499 126 126 126 222 222 222 F F F
145932024-02-20T22:47:22.110Z 25 500-519 134 134 134 238 238 238 F F F
145942024-02-20T22:47:22.110Z 26 520-539 134 134 134 238 238 238 F F F
145952024-02-20T22:47:22.110Z 27 540-559 129 129 129 226 226 226 F F F
145962024-02-20T22:47:22.110Z 28 560-579 135 135 135 240 240 240 F F F
145972024-02-20T22:47:22.110Z 29 580-599 123 123 123 212 212 212 F F F
145982024-02-20T22:47:22.110ZMax gen: 135, Max flush: 240
145992024-02-20T22:47:22.111Z
146002024-02-20T22:47:22.111ZBegin loop 68
146012024-02-20T22:47:22.111ZDownstairs to restart: 2
146022024-02-20T22:47:22.111Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560080367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682}
146032024-02-20T22:47:22.111Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"Crucible runtime is spawned
146042024-02-20T22:47:22.111Z2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560488912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682}
146052024-02-20T22:47:22.111Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.5605292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682}
146062024-02-20T22:47:22.111Z{"msg":"Crucible 65308f7e-a5de-4ec6-a0b8-6678795fb62b has session id: 6398bad5-1953-4bf2-959b-134df1f71b19","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560557279Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146072024-02-20T22:47:22.111Z{"msg":"Upstairs opts: Upstairs UUID: 65308f7e-a5de-4ec6-a0b8-6678795fb62b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560617537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146082024-02-20T22:47:22.111Z{"msg":"Crucible stats registered with UUID: 65308f7e-a5de-4ec6-a0b8-6678795fb62b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560660666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146092024-02-20T22:47:22.111Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560703344Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682}
146102024-02-20T22:47:22.111Z{"msg":"65308f7e-a5de-4ec6-a0b8-6678795fb62b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560723273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146112024-02-20T22:47:22.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560762102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146122024-02-20T22:47:22.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.56080489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146132024-02-20T22:47:22.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560837359Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146142024-02-20T22:47:22.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560883228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146152024-02-20T22:47:22.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560933146Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146162024-02-20T22:47:22.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.560983184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146172024-02-20T22:47:22.111Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561029022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146182024-02-20T22:47:22.111Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.56107876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146192024-02-20T22:47:22.112Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561111349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146202024-02-20T22:47:22.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561140508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146212024-02-20T22:47:22.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561187896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146222024-02-20T22:47:22.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561209725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146232024-02-20T22:47:22.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561668119Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146242024-02-20T22:47:22.112Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561711967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146252024-02-20T22:47:22.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561773905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146262024-02-20T22:47:22.112Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561803514Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146272024-02-20T22:47:22.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561857642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146282024-02-20T22:47:22.112Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.561885151Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146292024-02-20T22:47:22.112Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562348894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146302024-02-20T22:47:22.112Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562502629Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146312024-02-20T22:47:22.112Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.56274054Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146322024-02-20T22:47:22.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562895015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146332024-02-20T22:47:22.112Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562936404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146342024-02-20T22:47:22.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562953823Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146352024-02-20T22:47:22.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562967522Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146362024-02-20T22:47:22.112Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.562989122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146372024-02-20T22:47:22.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563002961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146382024-02-20T22:47:22.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563172095Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146392024-02-20T22:47:22.112Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563217473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146402024-02-20T22:47:22.112Z{"msg":"[0]R flush_numbers[0..12]: [240, 240, 240, 224, 240, 234, 233, 240, 238, 232, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563245002Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146412024-02-20T22:47:22.112Z{"msg":"[0]R generation[0..12]: [135, 135, 135, 127, 135, 132, 132, 135, 134, 131, 134, 135]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563280761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146422024-02-20T22:47:22.112Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.56331592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146432024-02-20T22:47:22.112Z{"msg":"[1]R flush_numbers[0..12]: [240, 240, 240, 224, 240, 234, 233, 240, 238, 232, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563354749Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146442024-02-20T22:47:22.113ZWait for a query_work_queue command to finish before sending IO
146452024-02-20T22:47:22.113Z{"msg":"[1]R generation[0..12]: [135, 135, 135, 127, 135, 132, 132, 135, 134, 131, 134, 135]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563397287Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146462024-02-20T22:47:22.113Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563438815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146472024-02-20T22:47:22.113Z{"msg":"[2]R flush_numbers[0..12]: [240, 240, 240, 224, 240, 234, 233, 240, 238, 232, 238, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563479794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146482024-02-20T22:47:22.113Z{"msg":"[2]R generation[0..12]: [135, 135, 135, 127, 135, 132, 132, 135, 134, 131, 134, 135]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563519892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146492024-02-20T22:47:22.113Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563558481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146502024-02-20T22:47:22.113Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.56358726Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146512024-02-20T22:47:22.113Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.56360205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146522024-02-20T22:47:22.113Z{"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563618519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146532024-02-20T22:47:22.113Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563643618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146542024-02-20T22:47:22.113Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563657788Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146552024-02-20T22:47:22.113Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563670187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146562024-02-20T22:47:22.113Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563691017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146572024-02-20T22:47:22.113Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563704756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146582024-02-20T22:47:22.113Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563718096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146592024-02-20T22:47:22.113Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563730805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146602024-02-20T22:47:22.113Z{"msg":"65308f7e-a5de-4ec6-a0b8-6678795fb62b is now active with session: 6398bad5-1953-4bf2-959b-134df1f71b19","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563751574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146612024-02-20T22:47:22.113Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563763834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682,"session_id":"6398bad5-1953-4bf2-959b-134df1f71b19"}
146622024-02-20T22:47:22.113Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:19.563776213Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682}
146632024-02-20T22:47:22.113ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146642024-02-20T22:47:22.113ZLoading write count information from file "/tmp/test_repair_verify.data"
146652024-02-20T22:47:22.113ZRun Repair workload
146662024-02-20T22:47:22.114Z01/30 Flush
146672024-02-20T22:47:22.125Z02/30 Write block 221 len 20480 data: 8 9 9 8 7
146682024-02-20T22:47:22.125Z03/30 Read block 89 len 24576
146692024-02-20T22:47:22.126Z04/30 Read block 156 len 32768
146702024-02-20T22:47:22.126Z05/30 Write block 209 len 28672 data: 11 13 12 10 5 5 6
146712024-02-20T22:47:22.126Z06/30 Write block 358 len 36864 data: 7 12 14 13 15 17 15 17 16
146722024-02-20T22:47:22.126Z07/30 Read block 130 len 24576
146732024-02-20T22:47:22.126Z08/30 Read block 181 len 16384
146742024-02-20T22:47:22.126Z09/30 Read block 566 len 36864
146752024-02-20T22:47:22.126Z10/30 Read block 36 len 16384
146762024-02-20T22:47:22.126Z11/30 Read block 264 len 12288
146772024-02-20T22:47:22.126Z12/30 Write block 197 len 16384 data: 9 8 9 6
146782024-02-20T22:47:22.126Z13/30 Write block 598 len 8192 data: 10 4
146792024-02-20T22:47:22.126Z14/30 Read block 15 len 32768
146802024-02-20T22:47:22.126Z15/30 Write block 373 len 20480 data: 12 9 9 10 13
146812024-02-20T22:47:22.126Z16/30 Read block 497 len 36864
146822024-02-20T22:47:22.127Z17/30 Read block 265 len 20480
146832024-02-20T22:47:22.127Z18/30 Read block 548 len 8192
146842024-02-20T22:47:22.127Z19/30 Write block 56 len 28672 data: 8 7 7 10 8 7 8
146852024-02-20T22:47:22.127Z20/30 Write block 596 len 12288 data: 13 12 11
146862024-02-20T22:47:22.127Z21/30 Read block 138 len 4096
146872024-02-20T22:47:22.127Z22/30 Read block 423 len 28672
146882024-02-20T22:47:22.127Z23/30 Write block 15 len 4096 data: 8
146892024-02-20T22:47:22.127Z24/30 Write block 299 len 20480 data: 12 12 15 17 17
146902024-02-20T22:47:22.127Z25/30 Write block 362 len 12288 data: 16 18 16
146912024-02-20T22:47:22.127Z26/30 Write block 41 len 32768 data: 9 9 9 9 7 7 8 10
146922024-02-20T22:47:22.127Z27/30 Write block 482 len 36864 data: 17 17 15 13 10 9 9 7 7
146932024-02-20T22:47:22.127Z28/30 Write block 340 len 36864 data: 14 14 12 12 9 9 9 11 12
146942024-02-20T22:47:22.127Z29/30 Read block 339 len 20480
146952024-02-20T22:47:22.127Z30/30 Write block 224 len 4096 data: 9
146962024-02-20T22:47:22.127Z----------------------------------------------------------------
146972024-02-20T22:47:22.127Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:29
146982024-02-20T22:47:22.127ZStates: Active Active Active
146992024-02-20T22:47:22.127ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147002024-02-20T22:47:22.127Z 2 Acked 1001 Write 5 Done Done Done false
147012024-02-20T22:47:22.127Z 3 Acked 1002 Read 6 Done Done Done false
147022024-02-20T22:47:22.127Z 4 Acked 1003 Read 8 Done Done Sent false
147032024-02-20T22:47:22.127Z 5 Acked 1004 Write 7 Done Done Sent false
147042024-02-20T22:47:22.128Z 6 Acked 1005 Write 9 Done Done Sent false
147052024-02-20T22:47:22.128Z 7 Acked 1006 Read 6 Done Done Sent false
147062024-02-20T22:47:22.128Z 8 Acked 1007 Read 4 Done Done Sent false
147072024-02-20T22:47:22.128Z 9 Acked 1008 Read 9 Done Done Sent false
147082024-02-20T22:47:22.128Z 10 Acked 1009 Read 4 Done Done Sent false
147092024-02-20T22:47:22.128Z 11 Acked 1010 Read 3 Done Done Sent false
147102024-02-20T22:47:22.128Z 12 Acked 1011 Write 4 Done Done Sent false
147112024-02-20T22:47:22.128Z 13 Acked 1012 Write 2 Done Done Sent false
147122024-02-20T22:47:22.128Z 14 Acked 1013 Read 8 Done Done Sent false
147132024-02-20T22:47:22.128Z 15 Acked 1014 Write 5 Done Done Sent false
147142024-02-20T22:47:22.128Z 16 Acked 1015 Read 9 Done Done Sent false
147152024-02-20T22:47:22.128Z 17 Acked 1016 Read 5 Done Done Sent false
147162024-02-20T22:47:22.128Z 18 Acked 1017 Read 2 Done Done Sent false
147172024-02-20T22:47:22.128Z 19 Acked 1018 Write 7 Done Done Sent false
147182024-02-20T22:47:22.128Z 20 Acked 1019 Write 3 Done Done Sent false
147192024-02-20T22:47:22.128Z 21 Acked 1020 Read 1 Done Done Sent false
147202024-02-20T22:47:22.128Z 22 Acked 1021 Read 7 Done Done Sent false
147212024-02-20T22:47:22.128Z 23 Acked 1022 Write 1 Done Done Sent false
147222024-02-20T22:47:22.128Z 24 Acked 1023 Write 5 Done Done Sent false
147232024-02-20T22:47:22.128Z 25 Acked 1024 Write 3 Done Done Sent false
147242024-02-20T22:47:22.128Z 26 Acked 1025 Write 8 Done Done Sent false
147252024-02-20T22:47:22.128Z 27 Acked 1026 Write 9 Done Done Sent false
147262024-02-20T22:47:22.128Z 28 Acked 1027 Write 9 Done Done Sent false
147272024-02-20T22:47:22.128Z 29 Acked 1028 Read 5 Done Done Sent false
147282024-02-20T22:47:22.128Z 30 Acked 1029 Write 1 Sent Sent Sent false
147292024-02-20T22:47:22.129Z STATES DS:0 DS:1 DS:2 TOTAL
147302024-02-20T22:47:22.129Z New 0 0 0 0
147312024-02-20T22:47:22.129Z Sent 1 1 27 29
147322024-02-20T22:47:22.129Z Done 28 28 2 58
147332024-02-20T22:47:22.129Z Skipped 0 0 0 0
147342024-02-20T22:47:22.129Z Error 0 0 0 0
147352024-02-20T22:47:22.129ZLast Flush: 1000 1000 1000
147362024-02-20T22:47:22.129ZDownstairs last five completed: 1000
147372024-02-20T22:47:22.129ZUpstairs last five completed: 30 29 28 27 26
147382024-02-20T22:47:22.129Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:19.590034574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1682}
147392024-02-20T22:47:22.129ZWrote out file "/tmp/test_repair_verify.data"
147402024-02-20T22:47:22.129Z
147412024-02-20T22:47:22.129ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147422024-02-20T22:47:22.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147432024-02-20T22:47:22.129Z 0 000-019 135 135 135 240 240 240 T T F <---
147442024-02-20T22:47:22.129Z 1 020-039 135 135 135 240 240 240 F F F
147452024-02-20T22:47:22.129Z 2 040-059 135 135 135 240 240 240 T T F <---
147462024-02-20T22:47:22.129Z 3 060-079 127 127 127 224 224 224 T T F <---
147472024-02-20T22:47:22.129Z 4 080-099 135 135 135 240 240 240 F F F
147482024-02-20T22:47:22.129Z 5 100-119 132 132 132 234 234 234 F F F
147492024-02-20T22:47:22.129Z 6 120-139 132 132 132 233 233 233 F F F
147502024-02-20T22:47:22.129Z 7 140-159 135 135 135 240 240 240 F F F
147512024-02-20T22:47:22.129Z 8 160-179 134 134 134 238 238 238 F F F
147522024-02-20T22:47:22.129Z 9 180-199 131 131 131 232 232 232 T T F <---
147532024-02-20T22:47:22.129Z 10 200-219 134 134 134 238 238 238 T T F <---
147542024-02-20T22:47:22.130Z 11 220-239 135 135 135 240 240 240 T T T
147552024-02-20T22:47:22.130Z 12 240-259 134 134 134 239 239 239 F F F
147562024-02-20T22:47:22.130Z 13 260-279 134 134 134 239 239 239 F F F
147572024-02-20T22:47:22.130Z 14 280-299 130 130 130 229 229 229 T T F <---
147582024-02-20T22:47:22.130Z 15 300-319 135 135 135 240 240 240 T T F <---
147592024-02-20T22:47:22.130Z 16 320-339 135 135 135 240 240 240 F F F
147602024-02-20T22:47:22.130Z 17 340-359 133 133 133 237 237 237 T T F <---
147612024-02-20T22:47:22.130Z 18 360-379 130 130 130 229 229 229 T T F <---
147622024-02-20T22:47:22.130Z 19 380-399 126 126 126 222 222 222 F F F
147632024-02-20T22:47:22.130Z 20 400-419 130 130 130 231 231 231 F F F
147642024-02-20T22:47:22.130Z 21 420-439 133 133 133 237 237 237 F F F
147652024-02-20T22:47:22.130Z 22 440-459 135 135 135 240 240 240 F F F
147662024-02-20T22:47:22.130Z 23 460-479 126 126 126 222 222 222 F F F
147672024-02-20T22:47:22.148Z 24 480-499 126 126 126 222 222 222 T T F <---
147682024-02-20T22:47:22.148Z 25 500-519 134 134 134 238 238 238 F F F
147692024-02-20T22:47:22.148Z 26 520-539 134 134 134 238 238 238 F F F
147702024-02-20T22:47:22.148Z 27 540-559 129 129 129 226 226 226 F F F
147712024-02-20T22:47:22.148Z 28 560-579 135 135 135 240 240 240 F F F
147722024-02-20T22:47:22.148Z 29 580-599 123 123 123 212 212 212 T T F <---
147732024-02-20T22:47:22.148ZMax gen: 135, Max flush: 240
147742024-02-20T22:47:22.148ZError: Difference in extent metadata found!
147752024-02-20T22:47:22.148ZA Difference in extent metadata is expected here
147762024-02-20T22:47:22.148ZOn loop 68
147772024-02-20T22:47:22.149Z
147782024-02-20T22:47:22.149ZVerifying data now
147792024-02-20T22:47:24.766ZNo change after verify
147802024-02-20T22:47:24.766ZLoop: 68 Downstairs dump after verify (and repair):
147812024-02-20T22:47:24.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147822024-02-20T22:47:24.779Z 0 000-019 136 136 136 241 241 241 F F F
147832024-02-20T22:47:24.779Z 1 020-039 135 135 135 240 240 240 F F F
147842024-02-20T22:47:24.779Z 2 040-059 136 136 136 241 241 241 F F F
147852024-02-20T22:47:24.779Z 3 060-079 136 136 136 241 241 241 F F F
147862024-02-20T22:47:24.779Z 4 080-099 135 135 135 240 240 240 F F F
147872024-02-20T22:47:24.780Z 5 100-119 132 132 132 234 234 234 F F F
147882024-02-20T22:47:24.780Z 6 120-139 132 132 132 233 233 233 F F F
147892024-02-20T22:47:24.780Z 7 140-159 135 135 135 240 240 240 F F F
147902024-02-20T22:47:24.780Z 8 160-179 134 134 134 238 238 238 F F F
147912024-02-20T22:47:24.780Z 9 180-199 136 136 136 241 241 241 F F F
147922024-02-20T22:47:24.780Z 10 200-219 136 136 136 241 241 241 F F F
147932024-02-20T22:47:24.790Z 11 220-239 136 136 136 241 241 241 F F F
147942024-02-20T22:47:24.790Z 12 240-259 134 134 134 239 239 239 F F F
147952024-02-20T22:47:24.790Z 13 260-279 134 134 134 239 239 239 F F F
147962024-02-20T22:47:24.790Z 14 280-299 136 136 136 241 241 241 F F F
147972024-02-20T22:47:24.791Z 15 300-319 136 136 136 241 241 241 F F F
147982024-02-20T22:47:24.791Z 16 320-339 135 135 135 240 240 240 F F F
147992024-02-20T22:47:24.791Z 17 340-359 136 136 136 241 241 241 F F F
148002024-02-20T22:47:24.791Z 18 360-379 136 136 136 241 241 241 F F F
148012024-02-20T22:47:24.791Z 19 380-399 126 126 126 222 222 222 F F F
148022024-02-20T22:47:24.791Z 20 400-419 130 130 130 231 231 231 F F F
148032024-02-20T22:47:24.791Z 21 420-439 133 133 133 237 237 237 F F F
148042024-02-20T22:47:24.791Z 22 440-459 135 135 135 240 240 240 F F F
148052024-02-20T22:47:24.791Z 23 460-479 126 126 126 222 222 222 F F F
148062024-02-20T22:47:24.791Z 24 480-499 136 136 136 241 241 241 F F F
148072024-02-20T22:47:24.791Z 25 500-519 134 134 134 238 238 238 F F F
148082024-02-20T22:47:24.791Z 26 520-539 134 134 134 238 238 238 F F F
148092024-02-20T22:47:24.791Z 27 540-559 129 129 129 226 226 226 F F F
148102024-02-20T22:47:24.791Z 28 560-579 135 135 135 240 240 240 F F F
148112024-02-20T22:47:24.791Z 29 580-599 136 136 136 241 241 241 F F F
148122024-02-20T22:47:24.791ZMax gen: 136, Max flush: 241
148132024-02-20T22:47:24.791Z
148142024-02-20T22:47:24.791ZBegin loop 69
148152024-02-20T22:47:24.791ZDownstairs to restart: 2
148162024-02-20T22:47:24.800Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.313826013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690}
148172024-02-20T22:47:24.800Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"Crucible runtime is spawned
148182024-02-20T22:47:24.809Zname":"crucible","level":30,"time":"2024-02-20T22:47:24.314222469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690}
148192024-02-20T22:47:24.809Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314265097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690}
148202024-02-20T22:47:24.810Z{"msg":"Crucible 97d1fb73-75f3-4f30-97ce-4cd9e0d3d5d6 has session id: dc846da8-9c3c-4894-b5d4-58f837d8179f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314287676Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148212024-02-20T22:47:24.810Z{"msg":"Upstairs opts: Upstairs UUID: 97d1fb73-75f3-4f30-97ce-4cd9e0d3d5d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314327675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148222024-02-20T22:47:24.810Z{"msg":"Crucible stats registered with UUID: 97d1fb73-75f3-4f30-97ce-4cd9e0d3d5d6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314361174Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148232024-02-20T22:47:24.810Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314396232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690}
148242024-02-20T22:47:24.810Z{"msg":"97d1fb73-75f3-4f30-97ce-4cd9e0d3d5d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314416912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148252024-02-20T22:47:24.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314448371Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148262024-02-20T22:47:24.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31447294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148272024-02-20T22:47:24.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314496209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148282024-02-20T22:47:24.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314519128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148292024-02-20T22:47:24.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314560247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148302024-02-20T22:47:24.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314599385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148312024-02-20T22:47:24.810Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314630214Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148322024-02-20T22:47:24.810Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314673053Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148332024-02-20T22:47:24.810Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314701212Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148342024-02-20T22:47:24.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31474463Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148352024-02-20T22:47:24.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314785258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148362024-02-20T22:47:24.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.314801968Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148372024-02-20T22:47:24.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.315422696Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148382024-02-20T22:47:24.810Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.315471594Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148392024-02-20T22:47:24.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.315502343Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148402024-02-20T22:47:24.810Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.315536102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148412024-02-20T22:47:24.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.315563281Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148422024-02-20T22:47:24.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31559568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148432024-02-20T22:47:24.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.316295524Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148442024-02-20T22:47:24.811Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.316358352Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148452024-02-20T22:47:24.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31642293Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148462024-02-20T22:47:24.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317038108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148472024-02-20T22:47:24.811Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317086656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148482024-02-20T22:47:24.811Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317119105Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148492024-02-20T22:47:24.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317166173Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148502024-02-20T22:47:24.811Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317207982Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148512024-02-20T22:47:24.811Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317232581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148522024-02-20T22:47:24.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31725446Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148532024-02-20T22:47:24.811Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317277809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148542024-02-20T22:47:24.811Z{"msg":"[0]R flush_numbers[0..12]: [241, 240, 241, 241, 240, 234, 233, 240, 238, 241, 241, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317305268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148552024-02-20T22:47:24.811Z{"msg":"[0]R generation[0..12]: [136, 135, 136, 136, 135, 132, 132, 135, 134, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317325978Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148562024-02-20T22:47:24.811Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317352837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148572024-02-20T22:47:24.811Z{"msg":"[1]R flush_numbers[0..12]: [241, 240, 241, 241, 240, 234, 233, 240, 238, 241, 241, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317380066Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148582024-02-20T22:47:24.811Z{"msg":"[1]R generation[0..12]: [136, 135, 136, 136, 135, 132, 132, 135, 134, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317401055Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148592024-02-20T22:47:24.811Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317427734Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148602024-02-20T22:47:24.811Z{"msg":"[2]R flush_numbers[0..12]: [241, 240, 241, 241, 240, 234, 233, 240, 238, 241, 241, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317459903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148612024-02-20T22:47:24.811Z{"msg":"[2]R generation[0..12]: [136, 135, 136, 136, 135, 132, 132, 135, 134, 136, 136, 136]","v":Wait for a query_work_queue command to finish before sending IO
148622024-02-20T22:47:24.811Z0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317509241Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148632024-02-20T22:47:24.811Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31754937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148642024-02-20T22:47:24.811Z{"msg":"Max found gen is 137","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317584718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148652024-02-20T22:47:24.812Z{"msg":"Generation requested: 138 >= found:137","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317632187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148662024-02-20T22:47:24.812Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317679125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148672024-02-20T22:47:24.812Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317717924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148682024-02-20T22:47:24.812Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317771102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148692024-02-20T22:47:24.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317800321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148702024-02-20T22:47:24.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317839039Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148712024-02-20T22:47:24.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317886447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148722024-02-20T22:47:24.812Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.317912597Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148732024-02-20T22:47:24.812Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148742024-02-20T22:47:24.812Z,"level":30,"time":"2024-02-20T22:47:24.317955195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148752024-02-20T22:47:24.812Z{"msg":"97d1fb73-75f3-4f30-97ce-4cd9e0d3d5d6 is now active with session: dc846da8-9c3c-4894-b5d4-58f837d8179f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.318012293Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148762024-02-20T22:47:24.812Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.318063901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690,"session_id":"dc846da8-9c3c-4894-b5d4-58f837d8179f"}
148772024-02-20T22:47:24.812Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:24.31811143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690}
148782024-02-20T22:47:24.812ZLoading write count information from file "/tmp/test_repair_verify.data"
148792024-02-20T22:47:24.812ZRun Repair workload
148802024-02-20T22:47:24.824Z01/30 Write block 210 len 16384 data: 14 13 11 6
148812024-02-20T22:47:24.824Z02/30 Read block 444 len 16384
148822024-02-20T22:47:24.824Z03/30 Read block 65 len 32768
148832024-02-20T22:47:24.824Z04/30 Read block 392 len 12288
148842024-02-20T22:47:24.824Z05/30 Read block 557 len 16384
148852024-02-20T22:47:24.824Z06/30 Read block 577 len 4096
148862024-02-20T22:47:24.824Z07/30 Write block 192 len 28672 data: 8 7 7 9 8 10 9
148872024-02-20T22:47:24.824Z08/30 Write block 404 len 36864 data: 13 12 12 10 9 11 12 14 14
148882024-02-20T22:47:24.824Z09/30 Read block 130 len 32768
148892024-02-20T22:47:24.824Z10/30 Write block 433 len 16384 data: 5 6 5 5
148902024-02-20T22:47:24.824Z11/30 Read block 413 len 4096
148912024-02-20T22:47:24.824Z12/30 Read block 91 len 16384
148922024-02-20T22:47:24.824Z13/30 Flush
148932024-02-20T22:47:24.824Z14/30 Write block 489 len 24576 data: 8 8 9 11 13 12
148942024-02-20T22:47:24.824Z15/30 Write block 74 len 4096 data: 10
148952024-02-20T22:47:24.824Z16/30 Read block 153 len 8192
148962024-02-20T22:47:24.824Z17/30 Write block 378 len 8192 data: 11 9
148972024-02-20T22:47:24.824Z18/30 Read block 317 len 12288
148982024-02-20T22:47:24.837Z19/30 Write block 320 len 40960 data: 15 13 15 11 9 14 15 13 13 12
148992024-02-20T22:47:24.837Z20/30 Flush
149002024-02-20T22:47:24.838Z21/30 Flush
149012024-02-20T22:47:24.838Z22/30 Write block 372 len 12288 data: 16 13 10
149022024-02-20T22:47:24.838Z23/30 Write block 48 len 40960 data: 11 12 10 10 11 8 11 9 9 8
149032024-02-20T22:47:24.838Z24/30 Write block 496 len 12288 data: 12 13 11
149042024-02-20T22:47:24.838Z25/30 Read block 356 len 4096
149052024-02-20T22:47:24.838Z26/30 Write block 65 len 36864 data: 10 9 13 11 12 14 14 13 11
149062024-02-20T22:47:24.838Z27/30 Read block 110 len 28672
149072024-02-20T22:47:24.838Z28/30 Write block 250 len 12288 data: 9 9 9
149082024-02-20T22:47:24.838Z29/30 Read block 347 len 32768
149092024-02-20T22:47:24.838Z30/30 Read block 67 len 4096
149102024-02-20T22:47:24.838Z----------------------------------------------------------------
149112024-02-20T22:47:24.838Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
149122024-02-20T22:47:24.838ZStates: Active Active Active
149132024-02-20T22:47:24.838ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149142024-02-20T22:47:24.838Z 1 Acked 1000 Write 4 Done Done Done false
149152024-02-20T22:47:24.838Z 2 Acked 1001 Read 4 Done Done Done false
149162024-02-20T22:47:24.838Z 3 Acked 1002 Read 8 Done Done Sent false
149172024-02-20T22:47:24.838Z 4 Acked 1003 Read 3 Done Done Sent false
149182024-02-20T22:47:24.838Z 5 Acked 1004 Read 4 Done Done Sent false
149192024-02-20T22:47:24.838Z 6 Acked 1005 Read 1 Done Done Sent false
149202024-02-20T22:47:24.838Z 7 Acked 1006 Write 7 Done Done Sent false
149212024-02-20T22:47:24.838Z 8 Acked 1007 Write 9 Done Done Sent false
149222024-02-20T22:47:24.838Z 9 Acked 1008 Read 8 Done Done Sent false
149232024-02-20T22:47:24.839Z 10 Acked 1009 Write 4 Done Done Sent false
149242024-02-20T22:47:24.839Z 11 Acked 1010 Read 1 Done Done Sent false
149252024-02-20T22:47:24.839Z 12 Acked 1011 Read 4 Done Done Sent false
149262024-02-20T22:47:24.839Z 13 Acked 1012 Flush 0 Done Done Sent false
149272024-02-20T22:47:24.839Z 14 Acked 1013 Write 6 Done Done Sent false
149282024-02-20T22:47:24.839Z 15 Acked 1014 Write 1 Done Done Sent false
149292024-02-20T22:47:24.839Z 16 Acked 1015 Read 2 Done Done Sent false
149302024-02-20T22:47:24.839Z 17 Acked 1016 Write 2 Done Done Sent false
149312024-02-20T22:47:24.839Z 18 Acked 1017 Read 3 Done Done Sent false
149322024-02-20T22:47:24.839Z 19 Acked 1018 Write 10 Done Done Sent false
149332024-02-20T22:47:24.839Z 20 Acked 1019 Flush 0 Done Done Sent false
149342024-02-20T22:47:24.839Z 21 Acked 1020 Flush 0 Done Done Sent false
149352024-02-20T22:47:24.839Z 22 Acked 1021 Write 3 Done Done Sent false
149362024-02-20T22:47:24.839Z 23 Acked 1022 Write 10 Done Done Sent false
149372024-02-20T22:47:24.839Z 24 Acked 1023 Write 3 Done Done Sent false
149382024-02-20T22:47:24.839Z 25 Acked 1024 Read 1 Done Done Sent false
149392024-02-20T22:47:24.839Z 26 Acked 1025 Write 9 Done Done Sent false
149402024-02-20T22:47:24.839Z 27 Acked 1026 Read 7 Done Done Sent false
149412024-02-20T22:47:24.839Z 28 Acked 1027 Write 3 Done Done Sent false
149422024-02-20T22:47:24.839Z 29 Acked 1028 Read 8 Done Done Sent false
149432024-02-20T22:47:24.839Z 30 Acked 1029 Read 1 Done Done Sent false
149442024-02-20T22:47:24.839Z STATES DS:0 DS:1 DS:2 TOTAL
149452024-02-20T22:47:24.839Z New 0 0 0 0
149462024-02-20T22:47:24.839Z Sent 0 0 28 28
149472024-02-20T22:47:24.839Z Done 30 30 2 62
149482024-02-20T22:47:24.840Z Skipped 0 0 0 0
149492024-02-20T22:47:24.840Z Error 0 0 0 0
149502024-02-20T22:47:24.840ZLast Flush: 1020 1020 0
149512024-02-20T22:47:24.840ZDownstairs last five completed:
149522024-02-20T22:47:24.840ZUpstairs last five completed: 30 29 28 27 26
149532024-02-20T22:47:24.840Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:24.349245796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1690}
149542024-02-20T22:47:24.840ZWrote out file "/tmp/test_repair_verify.data"
149552024-02-20T22:47:24.856Z
149562024-02-20T22:47:24.856ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149572024-02-20T22:47:24.864ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149582024-02-20T22:47:24.864Z 0 000-019 136 136 136 241 241 241 F F F
149592024-02-20T22:47:24.864Z 1 020-039 135 135 135 240 240 240 F F F
149602024-02-20T22:47:24.864Z 2 040-059 136 136 136 241 241 241 T T F <---
149612024-02-20T22:47:24.864Z 3 060-079 138 138 136 243 243 241 T T F <---
149622024-02-20T22:47:24.864Z 4 080-099 135 135 135 240 240 240 F F F
149632024-02-20T22:47:24.864Z 5 100-119 132 132 132 234 234 234 F F F
149642024-02-20T22:47:24.865Z 6 120-139 132 132 132 233 233 233 F F F
149652024-02-20T22:47:24.865Z 7 140-159 135 135 135 240 240 240 F F F
149662024-02-20T22:47:24.865Z 8 160-179 134 134 134 238 238 238 F F F
149672024-02-20T22:47:24.865Z 9 180-199 138 138 136 242 242 241 F F F <---
149682024-02-20T22:47:24.865Z 10 200-219 138 138 136 242 242 241 F F T <---
149692024-02-20T22:47:24.865Z 11 220-239 136 136 136 241 241 241 F F F
149702024-02-20T22:47:24.865Z 12 240-259 134 134 134 239 239 239 T T F <---
149712024-02-20T22:47:24.865Z 13 260-279 134 134 134 239 239 239 F F F
149722024-02-20T22:47:24.865Z 14 280-299 136 136 136 241 241 241 F F F
149732024-02-20T22:47:24.865Z 15 300-319 136 136 136 241 241 241 F F F
149742024-02-20T22:47:24.865Z 16 320-339 138 138 135 243 243 240 F F F <---
149752024-02-20T22:47:24.865Z 17 340-359 136 136 136 241 241 241 F F F
149762024-02-20T22:47:24.865Z 18 360-379 138 138 136 243 243 241 T T F <---
149772024-02-20T22:47:24.865Z 19 380-399 126 126 126 222 222 222 F F F
149782024-02-20T22:47:24.865Z 20 400-419 138 138 130 242 242 231 F F F <---
149792024-02-20T22:47:24.865Z 21 420-439 138 138 133 242 242 237 F F F <---
149802024-02-20T22:47:24.865Z 22 440-459 135 135 135 240 240 240 F F F
149812024-02-20T22:47:24.865Z 23 460-479 126 126 126 222 222 222 F F F
149822024-02-20T22:47:24.865Z 24 480-499 138 138 136 243 243 241 T T F <---
149832024-02-20T22:47:24.865Z 25 500-519 134 134 134 238 238 238 F F F
149842024-02-20T22:47:24.865Z 26 520-539 134 134 134 238 238 238 F F F
149852024-02-20T22:47:24.865Z 27 540-559 129 129 129 226 226 226 F F F
149862024-02-20T22:47:24.865Z 28 560-579 135 135 135 240 240 240 F F F
149872024-02-20T22:47:24.865Z 29 580-599 136 136 136 241 241 241 F F F
149882024-02-20T22:47:24.866ZMax gen: 138, Max flush: 243
149892024-02-20T22:47:24.866ZError: Difference in extent metadata found!
149902024-02-20T22:47:24.866ZA Difference in extent metadata is expected here
149912024-02-20T22:47:24.866ZOn loop 69
149922024-02-20T22:47:24.866Z
149932024-02-20T22:47:24.873ZVerifying data now
149942024-02-20T22:47:29.377ZNo change after verify
149952024-02-20T22:47:29.378ZLoop: 69 Downstairs dump after verify (and repair):
149962024-02-20T22:47:29.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149972024-02-20T22:47:29.392Z 0 000-019 136 136 136 241 241 241 F F F
149982024-02-20T22:47:29.392Z 1 020-039 135 135 135 240 240 240 F F F
149992024-02-20T22:47:29.392Z 2 040-059 139 139 139 244 244 244 F F F
150002024-02-20T22:47:29.392Z 3 060-079 139 139 139 244 244 244 F F F
150012024-02-20T22:47:29.392Z 4 080-099 135 135 135 240 240 240 F F F
150022024-02-20T22:47:29.392Z 5 100-119 132 132 132 234 234 234 F F F
150032024-02-20T22:47:29.392Z 6 120-139 132 132 132 233 233 233 F F F
150042024-02-20T22:47:29.392Z 7 140-159 135 135 135 240 240 240 F F F
150052024-02-20T22:47:29.392Z 8 160-179 134 134 134 238 238 238 F F F
150062024-02-20T22:47:29.392Z 9 180-199 138 138 138 242 242 242 F F F
150072024-02-20T22:47:29.392Z 10 200-219 138 138 138 242 242 242 F F F
150082024-02-20T22:47:29.392Z 11 220-239 136 136 136 241 241 241 F F F
150092024-02-20T22:47:29.392Z 12 240-259 139 139 139 244 244 244 F F F
150102024-02-20T22:47:29.392Z 13 260-279 134 134 134 239 239 239 F F F
150112024-02-20T22:47:29.392Z 14 280-299 136 136 136 241 241 241 F F F
150122024-02-20T22:47:29.392Z 15 300-319 136 136 136 241 241 241 F F F
150132024-02-20T22:47:29.392Z 16 320-339 138 138 138 243 243 243 F F F
150142024-02-20T22:47:29.392Z 17 340-359 136 136 136 241 241 241 F F F
150152024-02-20T22:47:29.393Z 18 360-379 139 139 139 244 244 244 F F F
150162024-02-20T22:47:29.393Z 19 380-399 126 126 126 222 222 222 F F F
150172024-02-20T22:47:29.393Z 20 400-419 138 138 138 242 242 242 F F F
150182024-02-20T22:47:29.393Z 21 420-439 138 138 138 242 242 242 F F F
150192024-02-20T22:47:29.393Z 22 440-459 135 135 135 240 240 240 F F F
150202024-02-20T22:47:29.393Z 23 460-479 126 126 126 222 222 222 F F F
150212024-02-20T22:47:29.393Z 24 480-499 139 139 139 244 244 244 F F F
150222024-02-20T22:47:29.393Z 25 500-519 134 134 134 238 238 238 F F F
150232024-02-20T22:47:29.393Z 26 520-539 134 134 134 238 238 238 F F F
150242024-02-20T22:47:29.393Z 27 540-559 129 129 129 226 226 226 F F F
150252024-02-20T22:47:29.393Z 28 560-579 135 135 135 240 240 240 F F F
150262024-02-20T22:47:29.393Z 29 580-599 136 136 136 241 241 241 F F F
150272024-02-20T22:47:29.393ZMax gen: 139, Max flush: 244
150282024-02-20T22:47:29.393Z
150292024-02-20T22:47:29.393ZBegin loop 70
150302024-02-20T22:47:29.393ZDownstairs to restart: 2
150312024-02-20T22:47:29.409Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.915779251Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698}
150322024-02-20T22:47:29.409Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name"Crucible runtime is spawned
150332024-02-20T22:47:29.435Z:"crucible","level":30,"time":"2024-02-20T22:47:28.916199766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698}
150342024-02-20T22:47:29.435Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916262544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698}
150352024-02-20T22:47:29.435Z{"msg":"Crucible 31650941-7eea-47d0-ad89-4c84eee9a78c has session id: 7cae3628-737a-4d07-a6ce-e633dc3484b6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916306362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150362024-02-20T22:47:29.435Z{"msg":"Upstairs opts: Upstairs UUID: 31650941-7eea-47d0-ad89-4c84eee9a78c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916351211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150372024-02-20T22:47:29.435Z{"msg":"Crucible stats registered with UUID: 31650941-7eea-47d0-ad89-4c84eee9a78c","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916395369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150382024-02-20T22:47:29.435Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916436797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698}
150392024-02-20T22:47:29.435Z{"msg":"31650941-7eea-47d0-ad89-4c84eee9a78c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916463587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150402024-02-20T22:47:29.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916503475Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150412024-02-20T22:47:29.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916548813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150422024-02-20T22:47:29.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916600531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150432024-02-20T22:47:29.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.91664015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150442024-02-20T22:47:29.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916663359Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150452024-02-20T22:47:29.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916708798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150462024-02-20T22:47:29.435Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916742426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150472024-02-20T22:47:29.435Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916789935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150482024-02-20T22:47:29.435Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916848943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150492024-02-20T22:47:29.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916897861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150502024-02-20T22:47:29.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.91693265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150512024-02-20T22:47:29.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.916981508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150522024-02-20T22:47:29.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.917691592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150532024-02-20T22:47:29.435Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.917740891Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150542024-02-20T22:47:29.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.917782179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150552024-02-20T22:47:29.436Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.917813708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150562024-02-20T22:47:29.436Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.917890755Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150572024-02-20T22:47:29.436Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.917923304Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150582024-02-20T22:47:29.436Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.918417946Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150592024-02-20T22:47:29.436Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.918507723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150602024-02-20T22:47:29.436Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.918552892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150612024-02-20T22:47:29.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.918954837Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150622024-02-20T22:47:29.436Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919011015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150632024-02-20T22:47:29.436Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919044334Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150642024-02-20T22:47:29.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.91917209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150652024-02-20T22:47:29.436Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919207848Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150662024-02-20T22:47:29.436Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919224428Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150672024-02-20T22:47:29.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919237067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150682024-02-20T22:47:29.436Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919251337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150692024-02-20T22:47:29.436Z{"msg":"[0]R flush_numbers[0..12]: [241, 240, 244, 244, 240, 234, 233, 240, 238, 242, 242, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919274146Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150702024-02-20T22:47:29.436Z{"msg":"[0]R generation[0..12]: [136, 135, 139, 139, 135, 132, 132, 135, 134, 138, 138, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919288625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150712024-02-20T22:47:29.436Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919310034Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150722024-02-20T22:47:29.436Z{"msg":"[1]R flush_numbers[0..12]: [241, 240, 244, 244, 240, 234, 233, 240, 238, 242, 242, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919325074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150732024-02-20T22:47:29.436Z{"msg":"[1]R generation[0..12]: [136, 135, 139, 139, 135, 132, 132, 135, 134, 138, 138, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919352213Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150742024-02-20T22:47:29.436Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919366732Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150752024-02-20T22:47:29.436ZWait for a query_work_queue command to finish before sending IO
150762024-02-20T22:47:29.436Z{"msg":"[2]R flush_numbers[0..12]: [241, 240, 244, 244, 240, 234, 233, 240, 238, 242, 242, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919394262Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150772024-02-20T22:47:29.436Z{"msg":"[2]R generation[0..12]: [136, 135, 139, 139, 135, 132, 132, 135, 134, 138, 138, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919415021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150782024-02-20T22:47:29.437Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.91942978Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150792024-02-20T22:47:29.437Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919460339Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150802024-02-20T22:47:29.437Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919488128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150812024-02-20T22:47:29.437Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919502718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150822024-02-20T22:47:29.437Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919522947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150832024-02-20T22:47:29.437Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919537517Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150842024-02-20T22:47:29.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919561645Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150852024-02-20T22:47:29.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919592115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150862024-02-20T22:47:29.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919608734Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150872024-02-20T22:47:29.437Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919629683Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150882024-02-20T22:47:29.437Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919642543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150892024-02-20T22:47:29.437Z{"msg":"31650941-7eea-47d0-ad89-4c84eee9a78c is now active with session: 7cae3628-737a-4d07-a6ce-e633dc3484b6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919672892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150902024-02-20T22:47:29.437Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.919697931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698,"session_id":"7cae3628-737a-4d07-a6ce-e633dc3484b6"}
150912024-02-20T22:47:29.437Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:28.91971189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698}
150922024-02-20T22:47:29.437ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150932024-02-20T22:47:29.437ZLoading write count information from file "/tmp/test_repair_verify.data"
150942024-02-20T22:47:29.437ZRun Repair workload
150952024-02-20T22:47:29.437Z01/30 Write block 456 len 4096 data: 16
150962024-02-20T22:47:29.437Z02/30 Read block 537 len 8192
150972024-02-20T22:47:29.437Z03/30 Read block 207 len 20480
150982024-02-20T22:47:29.437Z04/30 Write block 507 len 8192 data: 10 9
150992024-02-20T22:47:29.437Z05/30 Read block 213 len 36864
151002024-02-20T22:47:29.437Z06/30 Write block 272 len 16384 data: 8 6 6 8
151012024-02-20T22:47:29.437Z07/30 Read block 545 len 36864
151022024-02-20T22:47:29.438Z08/30 Write block 458 len 28672 data: 18 17 15 17 17 18 15
151032024-02-20T22:47:29.438Z09/30 Write block 553 len 40960 data: 6 6 7 8 7 8 6 5 8 6
151042024-02-20T22:47:29.438Z10/30 Write block 56 len 32768 data: 10 9 8 11 9 8 9 9
151052024-02-20T22:47:29.460Z11/30 Write block 241 len 20480 data: 6 4 4 4 6
151062024-02-20T22:47:29.460Z12/30 Read block 5 len 8192
151072024-02-20T22:47:29.461Z13/30 Read block 265 len 40960
151082024-02-20T22:47:29.461Z14/30 Read block 518 len 28672
151092024-02-20T22:47:29.461Z15/30 Read block 146 len 40960
151102024-02-20T22:47:29.461Z16/30 Write block 212 len 20480 data: 12 7 6 7 6
151112024-02-20T22:47:29.461Z17/30 Read block 39 len 28672
151122024-02-20T22:47:29.461Z18/30 Read block 435 len 8192
151132024-02-20T22:47:29.461Z19/30 Write block 126 len 4096 data: 8
151142024-02-20T22:47:29.461Z20/30 Read block 370 len 20480
151152024-02-20T22:47:29.461Z21/30 Read block 304 len 4096
151162024-02-20T22:47:29.461Z22/30 Read block 530 len 20480
151172024-02-20T22:47:29.461Z23/30 Write block 389 len 12288 data: 9 10 9
151182024-02-20T22:47:29.461Z24/30 Write block 88 len 40960 data: 6 7 10 10 7 7 10 9 6 8
151192024-02-20T22:47:29.461Z25/30 Write block 369 len 20480 data: 13 17 20 17 14
151202024-02-20T22:47:29.461Z26/30 Read block 511 len 32768
151212024-02-20T22:47:29.461Z27/30 Read block 478 len 32768
151222024-02-20T22:47:29.461Z28/30 Read block 299 len 24576
151232024-02-20T22:47:29.461Z29/30 Write block 305 len 8192 data: 15 11
151242024-02-20T22:47:29.461Z30/30 Read block 519 len 12288
151252024-02-20T22:47:29.461Z----------------------------------------------------------------
151262024-02-20T22:47:29.461Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
151272024-02-20T22:47:29.461ZStates: Active Active Active
151282024-02-20T22:47:29.461ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151292024-02-20T22:47:29.461Z 1 Acked 1000 Write 1 Done Done Done false
151302024-02-20T22:47:29.461Z 2 Acked 1001 Read 2 Done Done Done false
151312024-02-20T22:47:29.462Z 3 Acked 1002 Read 5 Done Done Done false
151322024-02-20T22:47:29.462Z 4 Acked 1003 Write 2 Done Done Sent false
151332024-02-20T22:47:29.462Z 5 Acked 1004 Read 9 Done Done Sent false
151342024-02-20T22:47:29.462Z 6 Acked 1005 Write 4 Done Done Sent false
151352024-02-20T22:47:29.462Z 7 Acked 1006 Read 9 Done Done Sent false
151362024-02-20T22:47:29.462Z 8 Acked 1007 Write 7 Done Done Sent false
151372024-02-20T22:47:29.462Z 9 Acked 1008 Write 10 Done Done Sent false
151382024-02-20T22:47:29.462Z 10 Acked 1009 Write 8 Done Done Sent false
151392024-02-20T22:47:29.462Z 11 Acked 1010 Write 5 Done Done Sent false
151402024-02-20T22:47:29.462Z 12 Acked 1011 Read 2 Done Done Sent false
151412024-02-20T22:47:29.462Z 13 Acked 1012 Read 10 Done Done Sent false
151422024-02-20T22:47:29.462Z 14 Acked 1013 Read 7 Done Done Sent false
151432024-02-20T22:47:29.462Z 15 Acked 1014 Read 10 Done Done Sent false
151442024-02-20T22:47:29.462Z 16 Acked 1015 Write 5 Done Done Sent false
151452024-02-20T22:47:29.462Z 17 Acked 1016 Read 7 Done Done Sent false
151462024-02-20T22:47:29.462Z 18 Acked 1017 Read 2 Done Done Sent false
151472024-02-20T22:47:29.462Z 19 Acked 1018 Write 1 Done Done Sent false
151482024-02-20T22:47:29.462Z 20 Acked 1019 Read 5 Done Done Sent false
151492024-02-20T22:47:29.462Z 21 Acked 1020 Read 1 Done Done Sent false
151502024-02-20T22:47:29.462Z 22 Acked 1021 Read 5 Done Done Sent false
151512024-02-20T22:47:29.462Z 23 Acked 1022 Write 3 Done Done Sent false
151522024-02-20T22:47:29.462Z 24 Acked 1023 Write 10 Done Done Sent false
151532024-02-20T22:47:29.462Z 25 Acked 1024 Write 5 Done Done Sent false
151542024-02-20T22:47:29.462Z 26 Acked 1025 Read 8 Done Done Sent false
151552024-02-20T22:47:29.462Z 27 Acked 1026 Read 8 Done Done Sent false
151562024-02-20T22:47:29.463Z 28 Acked 1027 Read 6 Done Done Sent false
151572024-02-20T22:47:29.463Z 29 Acked 1028 Write 2 Done Done Sent false
151582024-02-20T22:47:29.463Z 30 Acked 1029 Read 3 Done Done Sent false
151592024-02-20T22:47:29.463Z STATES DS:0 DS:1 DS:2 TOTAL
151602024-02-20T22:47:29.463Z New 0 0 0 0
151612024-02-20T22:47:29.463Z Sent 0 0 27 27
151622024-02-20T22:47:29.463Z Done 30 30 3 63
151632024-02-20T22:47:29.463Z Skipped 0 0 0 0
151642024-02-20T22:47:29.463Z Error 0 0 0 0
151652024-02-20T22:47:29.463ZLast Flush: 0 0 0
151662024-02-20T22:47:29.463ZDownstairs last five completed:
151672024-02-20T22:47:29.463ZUpstairs last five completed: 30 29 28 27 26
151682024-02-20T22:47:29.463Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:28.946103716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1698}
151692024-02-20T22:47:29.463ZWrote out file "/tmp/test_repair_verify.data"
151702024-02-20T22:47:29.463Z
151712024-02-20T22:47:29.463ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151722024-02-20T22:47:29.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151732024-02-20T22:47:29.468Z 0 000-019 136 136 136 241 241 241 F F F
151742024-02-20T22:47:29.468Z 1 020-039 135 135 135 240 240 240 F F F
151752024-02-20T22:47:29.468Z 2 040-059 139 139 139 244 244 244 T T F <---
151762024-02-20T22:47:29.468Z 3 060-079 139 139 139 244 244 244 T T F <---
151772024-02-20T22:47:29.468Z 4 080-099 135 135 135 240 240 240 T T F <---
151782024-02-20T22:47:29.468Z 5 100-119 132 132 132 234 234 234 F F F
151792024-02-20T22:47:29.468Z 6 120-139 132 132 132 233 233 233 T T F <---
151802024-02-20T22:47:29.468Z 7 140-159 135 135 135 240 240 240 F F F
151812024-02-20T22:47:29.468Z 8 160-179 134 134 134 238 238 238 F F F
151822024-02-20T22:47:29.468Z 9 180-199 138 138 138 242 242 242 F F F
151832024-02-20T22:47:29.468Z 10 200-219 138 138 138 242 242 242 T T F <---
151842024-02-20T22:47:29.468Z 11 220-239 136 136 136 241 241 241 F F F
151852024-02-20T22:47:29.468Z 12 240-259 139 139 139 244 244 244 T T F <---
151862024-02-20T22:47:29.468Z 13 260-279 134 134 134 239 239 239 T T F <---
151872024-02-20T22:47:29.469Z 14 280-299 136 136 136 241 241 241 F F F
151882024-02-20T22:47:29.469Z 15 300-319 136 136 136 241 241 241 T T F <---
151892024-02-20T22:47:29.469Z 16 320-339 138 138 138 243 243 243 F F F
151902024-02-20T22:47:29.469Z 17 340-359 136 136 136 241 241 241 F F F
151912024-02-20T22:47:29.469Z 18 360-379 139 139 139 244 244 244 T T F <---
151922024-02-20T22:47:29.469Z 19 380-399 126 126 126 222 222 222 T T F <---
151932024-02-20T22:47:29.469Z 20 400-419 138 138 138 242 242 242 F F F
151942024-02-20T22:47:29.469Z 21 420-439 138 138 138 242 242 242 F F F
151952024-02-20T22:47:29.469Z 22 440-459 135 135 135 240 240 240 T T T
151962024-02-20T22:47:29.469Z 23 460-479 126 126 126 222 222 222 T T F <---
151972024-02-20T22:47:29.469Z 24 480-499 139 139 139 244 244 244 F F F
151982024-02-20T22:47:29.469Z 25 500-519 134 134 134 238 238 238 T T F <---
151992024-02-20T22:47:29.469Z 26 520-539 134 134 134 238 238 238 F F F
152002024-02-20T22:47:29.469Z 27 540-559 129 129 129 226 226 226 T T F <---
152012024-02-20T22:47:29.469Z 28 560-579 135 135 135 240 240 240 T T F <---
152022024-02-20T22:47:29.469Z 29 580-599 136 136 136 241 241 241 F F F
152032024-02-20T22:47:29.469ZMax gen: 139, Max flush: 244
152042024-02-20T22:47:29.469ZError: Difference in extent metadata found!
152052024-02-20T22:47:29.469ZA Difference in extent metadata is expected here
152062024-02-20T22:47:29.469ZOn loop 70
152072024-02-20T22:47:29.469Z
152082024-02-20T22:47:29.469ZVerifying data now
152092024-02-20T22:47:34.219ZNo change after verify
152102024-02-20T22:47:34.219ZLoop: 70 Downstairs dump after verify (and repair):
152112024-02-20T22:47:34.234ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152122024-02-20T22:47:34.234Z 0 000-019 136 136 136 241 241 241 F F F
152132024-02-20T22:47:34.234Z 1 020-039 135 135 135 240 240 240 F F F
152142024-02-20T22:47:34.234Z 2 040-059 140 140 140 245 245 245 F F F
152152024-02-20T22:47:34.234Z 3 060-079 140 140 140 245 245 245 F F F
152162024-02-20T22:47:34.234Z 4 080-099 140 140 140 245 245 245 F F F
152172024-02-20T22:47:34.234Z 5 100-119 132 132 132 234 234 234 F F F
152182024-02-20T22:47:34.234Z 6 120-139 140 140 140 245 245 245 F F F
152192024-02-20T22:47:34.234Z 7 140-159 135 135 135 240 240 240 F F F
152202024-02-20T22:47:34.234Z 8 160-179 134 134 134 238 238 238 F F F
152212024-02-20T22:47:34.234Z 9 180-199 138 138 138 242 242 242 F F F
152222024-02-20T22:47:34.234Z 10 200-219 140 140 140 245 245 245 F F F
152232024-02-20T22:47:34.238Z 11 220-239 136 136 136 241 241 241 F F F
152242024-02-20T22:47:34.238Z 12 240-259 140 140 140 245 245 245 F F F
152252024-02-20T22:47:34.238Z 13 260-279 140 140 140 245 245 245 F F F
152262024-02-20T22:47:34.239Z 14 280-299 136 136 136 241 241 241 F F F
152272024-02-20T22:47:34.239Z 15 300-319 140 140 140 245 245 245 F F F
152282024-02-20T22:47:34.239Z 16 320-339 138 138 138 243 243 243 F F F
152292024-02-20T22:47:34.239Z 17 340-359 136 136 136 241 241 241 F F F
152302024-02-20T22:47:34.239Z 18 360-379 140 140 140 245 245 245 F F F
152312024-02-20T22:47:34.239Z 19 380-399 140 140 140 245 245 245 F F F
152322024-02-20T22:47:34.239Z 20 400-419 138 138 138 242 242 242 F F F
152332024-02-20T22:47:34.239Z 21 420-439 138 138 138 242 242 242 F F F
152342024-02-20T22:47:34.239Z 22 440-459 140 140 140 245 245 245 F F F
152352024-02-20T22:47:34.239Z 23 460-479 140 140 140 245 245 245 F F F
152362024-02-20T22:47:34.239Z 24 480-499 139 139 139 244 244 244 F F F
152372024-02-20T22:47:34.239Z 25 500-519 140 140 140 245 245 245 F F F
152382024-02-20T22:47:34.239Z 26 520-539 134 134 134 238 238 238 F F F
152392024-02-20T22:47:34.239Z 27 540-559 140 140 140 245 245 245 F F F
152402024-02-20T22:47:34.239Z 28 560-579 140 140 140 245 245 245 F F F
152412024-02-20T22:47:34.239Z 29 580-599 136 136 136 241 241 241 F F F
152422024-02-20T22:47:34.239ZMax gen: 140, Max flush: 245
152432024-02-20T22:47:34.239Z
152442024-02-20T22:47:34.239ZBegin loop 71
152452024-02-20T22:47:34.239ZDownstairs to restart: 2
152462024-02-20T22:47:34.256Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769128524Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706}
152472024-02-20T22:47:34.256Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\"Crucible runtime is spawned
152482024-02-20T22:47:34.256Z,\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.76951921Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706}
152492024-02-20T22:47:34.264Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769564648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706}
152502024-02-20T22:47:34.264Z{"msg":"Crucible 6e49d543-7bb0-4e58-999d-46566cb6e59b has session id: 56999510-ea4e-4417-9c26-4f7735183517","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769608186Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152512024-02-20T22:47:34.264Z{"msg":"Upstairs opts: Upstairs UUID: 6e49d543-7bb0-4e58-999d-46566cb6e59b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769669224Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152522024-02-20T22:47:34.264Z{"msg":"Crucible stats registered with UUID: 6e49d543-7bb0-4e58-999d-46566cb6e59b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769712243Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152532024-02-20T22:47:34.264Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769740772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706}
152542024-02-20T22:47:34.264Z{"msg":"6e49d543-7bb0-4e58-999d-46566cb6e59b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.76977235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152552024-02-20T22:47:34.264Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769800429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152562024-02-20T22:47:34.264Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769835488Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152572024-02-20T22:47:34.264Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769865667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152582024-02-20T22:47:34.264Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769899466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152592024-02-20T22:47:34.264Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769926475Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152602024-02-20T22:47:34.264Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769954114Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152612024-02-20T22:47:34.264Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.769979453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152622024-02-20T22:47:34.264Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770029101Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152632024-02-20T22:47:34.264Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.77006143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152642024-02-20T22:47:34.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770099789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152652024-02-20T22:47:34.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770188386Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152662024-02-20T22:47:34.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770276292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152672024-02-20T22:47:34.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770753765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152682024-02-20T22:47:34.265Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770791914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152692024-02-20T22:47:34.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770808873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152702024-02-20T22:47:34.265Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770843652Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152712024-02-20T22:47:34.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770924779Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152722024-02-20T22:47:34.265Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.770944399Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152732024-02-20T22:47:34.265Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.771369883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152742024-02-20T22:47:34.265Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.771411912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152752024-02-20T22:47:34.265Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.771558046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152762024-02-20T22:47:34.265Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.771874015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152772024-02-20T22:47:34.265Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.771938923Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152782024-02-20T22:47:34.265Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772002601Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152792024-02-20T22:47:34.265Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772038959Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152802024-02-20T22:47:34.265Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772054759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152812024-02-20T22:47:34.265Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772077078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152822024-02-20T22:47:34.265Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772090518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152832024-02-20T22:47:34.265Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772111237Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152842024-02-20T22:47:34.265Z{"msg":"[0]R flush_numbers[0..12]: [241, 240, 245, 245, 245, 234, 245, 240, 238, 242, 245, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772125586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152852024-02-20T22:47:34.265Z{"msg":"[0]R generation[0..12]: [136, 135, 140, 140, 140, 132, 140, 135, 134, 138, 140, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772146076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152862024-02-20T22:47:34.265Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772160845Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152872024-02-20T22:47:34.265Z{"msg":"[1]R flush_numbers[0..12]: [241, 240, 245, 245, 245, 234, 245, 240, 238, 242, 245, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772181314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152882024-02-20T22:47:34.266Z{"msg":"[1]R generation[0..12]: [136, 135, 140, 140, 140, 132, 140, 135, 134, 138, 140, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772195534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152892024-02-20T22:47:34.266Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772216453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152902024-02-20T22:47:34.266Z{"msg":"[2]R flush_numbers[0..12]: [241, 240, 245, 245, 245, 234, 245, 240, 238, 242, 245, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772230942Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152912024-02-20T22:47:34.266Z{"msg":"[2]R generation[0..12]: [136, 135, 140, 140, 140, 132, 140, 135, 134, 138, 140, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772251062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152922024-02-20T22:47:34.266Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772265221Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairsWait for a query_work_queue command to finish before sending IO
152932024-02-20T22:47:34.266Z","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152942024-02-20T22:47:34.266Z{"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.77229162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152952024-02-20T22:47:34.266Z{"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.77230513Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152962024-02-20T22:47:34.266Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772324719Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152972024-02-20T22:47:34.266Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772338419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152982024-02-20T22:47:34.266Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772367318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
152992024-02-20T22:47:34.266Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772389987Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153002024-02-20T22:47:34.266Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772413896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153012024-02-20T22:47:34.266Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772437335Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153022024-02-20T22:47:34.266Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772469114Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153032024-02-20T22:47:34.266Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772502013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153042024-02-20T22:47:34.266Z{"msg":"6e49d543-7bb0-4e58-999d-46566cb6e59b is now active with session: 56999510-ea4e-4417-9c26-4f7735183517","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772535002Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153052024-02-20T22:47:34.266Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.772561991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706,"session_id":"56999510-ea4e-4417-9c26-4f7735183517"}
153062024-02-20T22:47:34.266Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:33.7725951Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706}
153072024-02-20T22:47:34.266ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153082024-02-20T22:47:34.266ZLoading write count information from file "/tmp/test_repair_verify.data"
153092024-02-20T22:47:34.266ZRun Repair workload
153102024-02-20T22:47:34.272Z01/30 Write block 183 len 8192 data: 12 13
153112024-02-20T22:47:34.272Z02/30 Read block 251 len 32768
153122024-02-20T22:47:34.272Z03/30 Read block 435 len 36864
153132024-02-20T22:47:34.272Z04/30 Write block 12 len 16384 data: 7 9 8 9
153142024-02-20T22:47:34.272Z05/30 Read block 104 len 24576
153152024-02-20T22:47:34.272Z06/30 Read block 33 len 32768
153162024-02-20T22:47:34.272Z07/30 Read block 162 len 12288
153172024-02-20T22:47:34.272Z08/30 Write block 486 len 12288 data: 11 10 10
153182024-02-20T22:47:34.272Z09/30 Read block 163 len 16384
153192024-02-20T22:47:34.272Z10/30 Read block 112 len 28672
153202024-02-20T22:47:34.272Z11/30 Write block 251 len 12288 data: 10 10 11
153212024-02-20T22:47:34.272Z12/30 Read block 507 len 12288
153222024-02-20T22:47:34.272Z13/30 Read block 575 len 36864
153232024-02-20T22:47:34.277Z14/30 Read block 329 len 36864
153242024-02-20T22:47:34.277Z15/30 Write block 58 len 16384 data: 9 12 10 9
153252024-02-20T22:47:34.277Z16/30 Write block 145 len 28672 data: 9 7 7 10 12 14 16
153262024-02-20T22:47:34.277Z17/30 Write block 467 len 28672 data: 12 14 16 12 10 11 10
153272024-02-20T22:47:34.277Z18/30 Write block 469 len 12288 data: 17 13 11
153282024-02-20T22:47:34.277Z19/30 Read block 206 len 36864
153292024-02-20T22:47:34.277Z20/30 Write block 162 len 28672 data: 8 7 6 9 10 7 8
153302024-02-20T22:47:34.277Z21/30 Write block 468 len 12288 data: 15 18 14
153312024-02-20T22:47:34.284Z22/30 Read block 542 len 40960
153322024-02-20T22:47:34.284Z23/30 Read block 274 len 16384
153332024-02-20T22:47:34.284Z24/30 Write block 135 len 40960 data: 12 9 8 8 9 10 10 12 13 11
153342024-02-20T22:47:34.289Z25/30 Read block 83 len 40960
153352024-02-20T22:47:34.289Z26/30 Write block 534 len 20480 data: 10 13 14 12 15
153362024-02-20T22:47:34.289Z27/30 Read block 535 len 20480
153372024-02-20T22:47:34.289Z28/30 Write block 242 len 20480 data: 5 5 5 7 8
153382024-02-20T22:47:34.289Z29/30 Read block 98 len 4096
153392024-02-20T22:47:34.289Z30/30 Read block 457 len 40960
153402024-02-20T22:47:34.290Z----------------------------------------------------------------
153412024-02-20T22:47:34.290Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153422024-02-20T22:47:34.290ZStates: Active Active Active
153432024-02-20T22:47:34.290ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153442024-02-20T22:47:34.290Z 1 Acked 1000 Write 2 Done Done Done false
153452024-02-20T22:47:34.290Z 2 Acked 1001 Read 8 Done Done Done false
153462024-02-20T22:47:34.290Z 3 Acked 1002 Read 9 Done Done Done false
153472024-02-20T22:47:34.290Z 4 Acked 1003 Write 4 Done Done Done false
153482024-02-20T22:47:34.290Z 5 Acked 1004 Read 6 Done Done Done false
153492024-02-20T22:47:34.290Z 6 Acked 1005 Read 8 Done Done Done false
153502024-02-20T22:47:34.290Z 7 Acked 1006 Read 3 Done Done Done false
153512024-02-20T22:47:34.290Z 8 Acked 1007 Write 3 Done Done Done false
153522024-02-20T22:47:34.290Z 9 Acked 1008 Read 4 Done Done Sent false
153532024-02-20T22:47:34.290Z 10 Acked 1009 Read 7 Done Done Sent false
153542024-02-20T22:47:34.290Z 11 Acked 1010 Write 3 Done Done Sent false
153552024-02-20T22:47:34.290Z 12 Acked 1011 Read 3 Done Done Sent false
153562024-02-20T22:47:34.290Z 13 Acked 1012 Read 9 Done Done Sent false
153572024-02-20T22:47:34.290Z 14 Acked 1013 Read 9 Done Done Sent false
153582024-02-20T22:47:34.290Z 15 Acked 1014 Write 4 Done Done Sent false
153592024-02-20T22:47:34.290Z 16 Acked 1015 Write 7 Done Done Sent false
153602024-02-20T22:47:34.290Z 17 Acked 1016 Write 7 Done Done Sent false
153612024-02-20T22:47:34.290Z 18 Acked 1017 Write 3 Done Done Sent false
153622024-02-20T22:47:34.290Z 19 Acked 1018 Read 9 Done Done Sent false
153632024-02-20T22:47:34.290Z 20 Acked 1019 Write 7 Done Done Sent false
153642024-02-20T22:47:34.290Z 21 Acked 1020 Write 3 Done Done Sent false
153652024-02-20T22:47:34.291Z 22 Acked 1021 Read 10 Done Done Sent false
153662024-02-20T22:47:34.291Z 23 Acked 1022 Read 4 Done Done Sent false
153672024-02-20T22:47:34.291Z 24 Acked 1023 Write 10 Done Done Sent false
153682024-02-20T22:47:34.291Z 25 Acked 1024 Read 10 Done Done Sent false
153692024-02-20T22:47:34.291Z 26 Acked 1025 Write 5 Done Done Sent false
153702024-02-20T22:47:34.291Z 27 Acked 1026 Read 5 Done Done Sent false
153712024-02-20T22:47:34.291Z 28 Acked 1027 Write 5 Done Done Sent false
153722024-02-20T22:47:34.291Z 29 Acked 1028 Read 1 Done Done Sent false
153732024-02-20T22:47:34.291Z 30 Acked 1029 Read 10 Done Done Sent false
153742024-02-20T22:47:34.291Z STATES DS:0 DS:1 DS:2 TOTAL
153752024-02-20T22:47:34.291Z New 0 0 0 0
153762024-02-20T22:47:34.291Z Sent 0 0 22 22
153772024-02-20T22:47:34.291Z Done 30 30 8 68
153782024-02-20T22:47:34.291Z Skipped 0 0 0 0
153792024-02-20T22:47:34.291Z Error 0 0 0 0
153802024-02-20T22:47:34.291ZLast Flush: 0 0 0
153812024-02-20T22:47:34.291ZDownstairs last five completed:
153822024-02-20T22:47:34.291ZUpstairs last five completed: 30 29 28 27 26
153832024-02-20T22:47:34.291Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:33.801566083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1706}
153842024-02-20T22:47:34.291ZWrote out file "/tmp/test_repair_verify.data"
153852024-02-20T22:47:34.295Z
153862024-02-20T22:47:34.295ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153872024-02-20T22:47:34.314ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153882024-02-20T22:47:34.314Z 0 000-019 136 136 136 241 241 241 T T T
153892024-02-20T22:47:34.314Z 1 020-039 135 135 135 240 240 240 F F F
153902024-02-20T22:47:34.314Z 2 040-059 140 140 140 245 245 245 T T F <---
153912024-02-20T22:47:34.314Z 3 060-079 140 140 140 245 245 245 T T F <---
153922024-02-20T22:47:34.314Z 4 080-099 140 140 140 245 245 245 F F F
153932024-02-20T22:47:34.314Z 5 100-119 132 132 132 234 234 234 F F F
153942024-02-20T22:47:34.314Z 6 120-139 140 140 140 245 245 245 T T F <---
153952024-02-20T22:47:34.314Z 7 140-159 135 135 135 240 240 240 T T F <---
153962024-02-20T22:47:34.318Z 8 160-179 134 134 134 238 238 238 T T F <---
153972024-02-20T22:47:34.318Z 9 180-199 138 138 138 242 242 242 T T T
153982024-02-20T22:47:34.318Z 10 200-219 140 140 140 245 245 245 F F F
153992024-02-20T22:47:34.318Z 11 220-239 136 136 136 241 241 241 F F F
154002024-02-20T22:47:34.318Z 12 240-259 140 140 140 245 245 245 T T F <---
154012024-02-20T22:47:34.318Z 13 260-279 140 140 140 245 245 245 F F F
154022024-02-20T22:47:34.318Z 14 280-299 136 136 136 241 241 241 F F F
154032024-02-20T22:47:34.319Z 15 300-319 140 140 140 245 245 245 F F F
154042024-02-20T22:47:34.319Z 16 320-339 138 138 138 243 243 243 F F F
154052024-02-20T22:47:34.319Z 17 340-359 136 136 136 241 241 241 F F F
154062024-02-20T22:47:34.319Z 18 360-379 140 140 140 245 245 245 F F F
154072024-02-20T22:47:34.319Z 19 380-399 140 140 140 245 245 245 F F F
154082024-02-20T22:47:34.319Z 20 400-419 138 138 138 242 242 242 F F F
154092024-02-20T22:47:34.319Z 21 420-439 138 138 138 242 242 242 F F F
154102024-02-20T22:47:34.319Z 22 440-459 140 140 140 245 245 245 F F F
154112024-02-20T22:47:34.319Z 23 460-479 140 140 140 245 245 245 T T F <---
154122024-02-20T22:47:34.319Z 24 480-499 139 139 139 244 244 244 T T T
154132024-02-20T22:47:34.319Z 25 500-519 140 140 140 245 245 245 F F F
154142024-02-20T22:47:34.319Z 26 520-539 134 134 134 238 238 238 T T F <---
154152024-02-20T22:47:34.319Z 27 540-559 140 140 140 245 245 245 F F F
154162024-02-20T22:47:34.319Z 28 560-579 140 140 140 245 245 245 F F F
154172024-02-20T22:47:34.319Z 29 580-599 136 136 136 241 241 241 F F F
154182024-02-20T22:47:34.319ZMax gen: 140, Max flush: 245
154192024-02-20T22:47:34.319ZError: Difference in extent metadata found!
154202024-02-20T22:47:34.319ZA Difference in extent metadata is expected here
154212024-02-20T22:47:34.319ZOn loop 71
154222024-02-20T22:47:34.319Z
154232024-02-20T22:47:34.323ZVerifying data now
154242024-02-20T22:47:39.886ZNo change after verify
154252024-02-20T22:47:39.886ZLoop: 71 Downstairs dump after verify (and repair):
154262024-02-20T22:47:39.901ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154272024-02-20T22:47:39.901Z 0 000-019 141 141 141 246 246 246 F F F
154282024-02-20T22:47:39.901Z 1 020-039 135 135 135 240 240 240 F F F
154292024-02-20T22:47:39.901Z 2 040-059 141 141 141 246 246 246 F F F
154302024-02-20T22:47:39.901Z 3 060-079 141 141 141 246 246 246 F F F
154312024-02-20T22:47:39.901Z 4 080-099 140 140 140 245 245 245 F F F
154322024-02-20T22:47:39.901Z 5 100-119 132 132 132 234 234 234 F F F
154332024-02-20T22:47:39.901Z 6 120-139 141 141 141 246 246 246 F F F
154342024-02-20T22:47:39.901Z 7 140-159 141 141 141 246 246 246 F F F
154352024-02-20T22:47:39.905Z 8 160-179 141 141 141 246 246 246 F F F
154362024-02-20T22:47:39.905Z 9 180-199 141 141 141 246 246 246 F F F
154372024-02-20T22:47:39.905Z 10 200-219 140 140 140 245 245 245 F F F
154382024-02-20T22:47:39.905Z 11 220-239 136 136 136 241 241 241 F F F
154392024-02-20T22:47:39.905Z 12 240-259 141 141 141 246 246 246 F F F
154402024-02-20T22:47:39.905Z 13 260-279 140 140 140 245 245 245 F F F
154412024-02-20T22:47:39.906Z 14 280-299 136 136 136 241 241 241 F F F
154422024-02-20T22:47:39.906Z 15 300-319 140 140 140 245 245 245 F F F
154432024-02-20T22:47:39.906Z 16 320-339 138 138 138 243 243 243 F F F
154442024-02-20T22:47:39.906Z 17 340-359 136 136 136 241 241 241 F F F
154452024-02-20T22:47:39.906Z 18 360-379 140 140 140 245 245 245 F F F
154462024-02-20T22:47:39.906Z 19 380-399 140 140 140 245 245 245 F F F
154472024-02-20T22:47:39.906Z 20 400-419 138 138 138 242 242 242 F F F
154482024-02-20T22:47:39.906Z 21 420-439 138 138 138 242 242 242 F F F
154492024-02-20T22:47:39.906Z 22 440-459 140 140 140 245 245 245 F F F
154502024-02-20T22:47:39.906Z 23 460-479 141 141 141 246 246 246 F F F
154512024-02-20T22:47:39.906Z 24 480-499 141 141 141 246 246 246 F F F
154522024-02-20T22:47:39.906Z 25 500-519 140 140 140 245 245 245 F F F
154532024-02-20T22:47:39.906Z 26 520-539 141 141 141 246 246 246 F F F
154542024-02-20T22:47:39.906Z 27 540-559 140 140 140 245 245 245 F F F
154552024-02-20T22:47:39.906Z 28 560-579 140 140 140 245 245 245 F F F
154562024-02-20T22:47:39.906Z 29 580-599 136 136 136 241 241 241 F F F
154572024-02-20T22:47:39.906ZMax gen: 141, Max flush: 246
154582024-02-20T22:47:39.906Z
154592024-02-20T22:47:39.906ZBegin loop 72
154602024-02-20T22:47:39.906ZDownstairs to restart: 0
154612024-02-20T22:47:39.923Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436060684Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714}
154622024-02-20T22:47:39.931Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436355083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714}
154632024-02-20T22:47:39.931Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
154642024-02-20T22:47:39.931Z,"time":"2024-02-20T22:47:39.43644258Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714}
154652024-02-20T22:47:39.931Z{"msg":"Crucible 6f7d9b19-f318-4fb8-9361-6e642fddf20b has session id: dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436515708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154662024-02-20T22:47:39.931Z{"msg":"Upstairs opts: Upstairs UUID: 6f7d9b19-f318-4fb8-9361-6e642fddf20b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436567266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154672024-02-20T22:47:39.931Z{"msg":"Crucible stats registered with UUID: 6f7d9b19-f318-4fb8-9361-6e642fddf20b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436588205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154682024-02-20T22:47:39.931Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436613864Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714}
154692024-02-20T22:47:39.931Z{"msg":"6f7d9b19-f318-4fb8-9361-6e642fddf20b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436628984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154702024-02-20T22:47:39.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436648593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154712024-02-20T22:47:39.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436695711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154722024-02-20T22:47:39.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.43672215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154732024-02-20T22:47:39.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436759839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154742024-02-20T22:47:39.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436779338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154752024-02-20T22:47:39.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436819197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154762024-02-20T22:47:39.931Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436850536Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154772024-02-20T22:47:39.931Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436896504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154782024-02-20T22:47:39.931Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436934723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154792024-02-20T22:47:39.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.436973981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154802024-02-20T22:47:39.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437040589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154812024-02-20T22:47:39.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437071418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154822024-02-20T22:47:39.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437655327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154832024-02-20T22:47:39.931Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437691846Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154842024-02-20T22:47:39.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437707915Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154852024-02-20T22:47:39.932Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437722294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154862024-02-20T22:47:39.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437736824Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154872024-02-20T22:47:39.932Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.437756273Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154882024-02-20T22:47:39.932Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.438322973Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154892024-02-20T22:47:39.932Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.438366251Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154902024-02-20T22:47:39.932Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.438457778Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154912024-02-20T22:47:39.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.43898394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154922024-02-20T22:47:39.932Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439021408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154932024-02-20T22:47:39.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439036358Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154942024-02-20T22:47:39.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439049967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154952024-02-20T22:47:39.932Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439071716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154962024-02-20T22:47:39.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439086376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154972024-02-20T22:47:39.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439105455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154982024-02-20T22:47:39.932Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439119035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
154992024-02-20T22:47:39.932Z{"msg":"[0]R flush_numbers[0..12]: [246, 240, 246, 246, 245, 234, 246, 246, 246, 246, 245, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439139784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155002024-02-20T22:47:39.932Z{"msg":"[0]R generation[0..12]: [141, 135, 141, 141, 140, 132, 141, 141, 141, 141, 140, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439154263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155012024-02-20T22:47:39.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439175122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155022024-02-20T22:47:39.932ZWait for a query_work_queue command to finish before sending IO
155032024-02-20T22:47:39.932Z{"msg":"[1]R flush_numbers[0..12]: [246, 240, 246, 246, 245, 234, 246, 246, 246, 246, 245, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439206292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155042024-02-20T22:47:39.932Z{"msg":"[1]R generation[0..12]: [141, 135, 141, 141, 140, 132, 141, 141, 141, 141, 140, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439220861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155052024-02-20T22:47:39.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.4392356Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155062024-02-20T22:47:39.932Z{"msg":"[2]R flush_numbers[0..12]: [246, 240, 246, 246, 245, 234, 246, 246, 246, 246, 245, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.43924966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155072024-02-20T22:47:39.932Z{"msg":"[2]R generation[0..12]: [141, 135, 141, 141, 140, 132, 141, 141, 141, 141, 140, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439270499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155082024-02-20T22:47:39.932Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439285429Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155092024-02-20T22:47:39.933Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439299538Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155102024-02-20T22:47:39.933Z{"msg":"Generation requested: 144 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439314218Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155112024-02-20T22:47:39.933Z{"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439327367Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155122024-02-20T22:47:39.933Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439348096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155132024-02-20T22:47:39.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439361976Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155142024-02-20T22:47:39.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439375166Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155152024-02-20T22:47:39.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439395365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155162024-02-20T22:47:39.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439408824Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155172024-02-20T22:47:39.933Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439422264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155182024-02-20T22:47:39.933Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439441613Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155192024-02-20T22:47:39.933Z{"msg":"6f7d9b19-f318-4fb8-9361-6e642fddf20b is now active with session: dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439454672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155202024-02-20T22:47:39.933Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439467372Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714,"session_id":"dd9b3cc7-0a9e-49a3-acbd-4f7b0fc377eb"}
155212024-02-20T22:47:39.933Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:39.439486202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714}
155222024-02-20T22:47:39.933ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155232024-02-20T22:47:39.937ZLoading write count information from file "/tmp/test_repair_verify.data"
155242024-02-20T22:47:39.937ZRun Repair workload
155252024-02-20T22:47:39.937Z01/30 Write block 572 len 32768 data: 4 8 8 7 9 13 16 18
155262024-02-20T22:47:39.937Z02/30 Write block 523 len 12288 data: 10 8 8
155272024-02-20T22:47:39.937Z03/30 Flush
155282024-02-20T22:47:39.937Z04/30 Write block 181 len 4096 data: 16
155292024-02-20T22:47:39.937Z05/30 Read block 124 len 8192
155302024-02-20T22:47:39.937Z06/30 Write block 195 len 4096 data: 10
155312024-02-20T22:47:39.937Z07/30 Write block 256 len 20480 data: 7 8 10 10 9
155322024-02-20T22:47:39.937Z08/30 Read block 360 len 40960
155332024-02-20T22:47:39.937Z09/30 Read block 433 len 24576
155342024-02-20T22:47:39.937Z10/30 Write block 538 len 28672 data: 16 13 14 14 11 12 10
155352024-02-20T22:47:39.937Z11/30 Write block 513 len 28672 data: 10 10 10 9 11 9 8
155362024-02-20T22:47:39.942Z12/30 Read block 121 len 28672
155372024-02-20T22:47:39.942Z13/30 Write block 306 len 4096 data: 12
155382024-02-20T22:47:39.942Z14/30 Read block 398 len 20480
155392024-02-20T22:47:39.942Z15/30 Write block 29 len 32768 data: 19 18 19 20 21 19 20 17
155402024-02-20T22:47:39.942Z16/30 Read block 469 len 28672
155412024-02-20T22:47:39.945Z17/30 Read block 35 len 24576
155422024-02-20T22:47:39.945Z18/30 Write block 210 len 24576 data: 15 14 13 8 7 8
155432024-02-20T22:47:39.945Z19/30 Flush
155442024-02-20T22:47:39.949Z20/30 Write block 445 len 12288 data: 10 10 10
155452024-02-20T22:47:39.949Z21/30 Read block 324 len 40960
155462024-02-20T22:47:39.949Z22/30 Write block 418 len 24576 data: 12 15 15 11 9 9
155472024-02-20T22:47:39.949Z23/30 Write block 30 len 16384 data: 19 20 21 22
155482024-02-20T22:47:39.953Z24/30 Read block 493 len 12288
155492024-02-20T22:47:39.953Z25/30 Read block 382 len 40960
155502024-02-20T22:47:39.953Z26/30 Read block 463 len 36864
155512024-02-20T22:47:39.956Z27/30 Write block 214 len 40960 data: 8 9 7 4 5 5 7 9 10 10
155522024-02-20T22:47:39.956Z28/30 Read block 363 len 24576
155532024-02-20T22:47:39.957Z29/30 Read block 589 len 32768
155542024-02-20T22:47:39.957Z30/30 Write block 579 len 28672 data: 19 18 15 15 13 9 8
155552024-02-20T22:47:39.961Z----------------------------------------------------------------
155562024-02-20T22:47:39.961Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:27
155572024-02-20T22:47:39.961ZStates: Active Active Active
155582024-02-20T22:47:39.961ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155592024-02-20T22:47:39.961Z 4 Acked 1003 Write 1 Sent Done Done false
155602024-02-20T22:47:39.961Z 5 Acked 1004 Read 2 Sent Done Done false
155612024-02-20T22:47:39.961Z 6 Acked 1005 Write 1 Sent Done Done false
155622024-02-20T22:47:39.961Z 7 Acked 1006 Write 5 Sent Done Done false
155632024-02-20T22:47:39.961Z 8 Acked 1007 Read 10 Sent Done Done false
155642024-02-20T22:47:39.961Z 9 Acked 1008 Read 6 Sent Done Done false
155652024-02-20T22:47:39.961Z 10 Acked 1009 Write 7 Sent Done Done false
155662024-02-20T22:47:39.961Z 11 Acked 1010 Write 7 Sent Done Done false
155672024-02-20T22:47:39.961Z 12 Acked 1011 Read 7 Sent Done Done false
155682024-02-20T22:47:39.961Z 13 Acked 1012 Write 1 Sent Done Done false
155692024-02-20T22:47:39.961Z 14 Acked 1013 Read 5 Sent Done Done false
155702024-02-20T22:47:39.961Z 15 Acked 1014 Write 8 Sent Done Done false
155712024-02-20T22:47:39.961Z 16 Acked 1015 Read 7 Sent Done Done false
155722024-02-20T22:47:39.961Z 17 Acked 1016 Read 6 Sent Done Done false
155732024-02-20T22:47:39.961Z 18 Acked 1017 Write 6 Sent Done Done false
155742024-02-20T22:47:39.961Z 19 Acked 1018 Flush 0 Sent Done Done false
155752024-02-20T22:47:39.961Z 20 Acked 1019 Write 3 Sent Done Done false
155762024-02-20T22:47:39.962Z 21 Acked 1020 Read 10 Sent Done Done false
155772024-02-20T22:47:39.962Z 22 Acked 1021 Write 6 Sent Done Done false
155782024-02-20T22:47:39.962Z 23 Acked 1022 Write 4 Sent Done Done false
155792024-02-20T22:47:39.962Z 24 Acked 1023 Read 3 Sent Done Done false
155802024-02-20T22:47:39.962Z 25 Acked 1024 Read 10 Sent Done Done false
155812024-02-20T22:47:39.962Z 26 Acked 1025 Read 9 Sent Done Done false
155822024-02-20T22:47:39.962Z 27 Acked 1026 Write 10 Sent Done Done false
155832024-02-20T22:47:39.962Z 28 Acked 1027 Read 6 Sent Done Done false
155842024-02-20T22:47:39.962Z 29 Acked 1028 Read 8 Sent Done Done false
155852024-02-20T22:47:39.962Z 30 Acked 1029 Write 7 Sent Sent Sent false
155862024-02-20T22:47:39.962Z STATES DS:0 DS:1 DS:2 TOTAL
155872024-02-20T22:47:39.962Z New 0 0 0 0
155882024-02-20T22:47:39.962Z Sent 27 1 1 29
155892024-02-20T22:47:39.962Z Done 0 26 26 52
155902024-02-20T22:47:39.962Z Skipped 0 0 0 0
155912024-02-20T22:47:39.962Z Error 0 0 0 0
155922024-02-20T22:47:39.962ZLast Flush: 1002 1018 1018
155932024-02-20T22:47:39.962ZDownstairs last five completed: 1002 1001 1000
155942024-02-20T22:47:39.962ZUpstairs last five completed: 30 29 28 27 26
155952024-02-20T22:47:39.962Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:39.470395396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1714}
155962024-02-20T22:47:39.962ZWrote out file "/tmp/test_repair_verify.data"
155972024-02-20T22:47:39.966Z
155982024-02-20T22:47:39.966ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155992024-02-20T22:47:39.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156002024-02-20T22:47:39.983Z 0 000-019 141 141 141 246 246 246 F F F
156012024-02-20T22:47:39.983Z 1 020-039 135 144 144 240 248 248 F T T <---
156022024-02-20T22:47:39.983Z 2 040-059 141 141 141 246 246 246 F F F
156032024-02-20T22:47:39.983Z 3 060-079 141 141 141 246 246 246 F F F
156042024-02-20T22:47:39.983Z 4 080-099 140 140 140 245 245 245 F F F
156052024-02-20T22:47:39.990Z 5 100-119 132 132 132 234 234 234 F F F
156062024-02-20T22:47:39.990Z 6 120-139 141 141 141 246 246 246 F F F
156072024-02-20T22:47:39.990Z 7 140-159 141 141 141 246 246 246 F F F
156082024-02-20T22:47:39.990Z 8 160-179 141 141 141 246 246 246 F F F
156092024-02-20T22:47:39.990Z 9 180-199 141 144 144 246 248 248 F F F <---
156102024-02-20T22:47:39.991Z 10 200-219 140 144 144 245 248 248 F T T <---
156112024-02-20T22:47:39.991Z 11 220-239 136 136 136 241 241 241 F T T <---
156122024-02-20T22:47:39.991Z 12 240-259 141 144 144 246 248 248 F F F <---
156132024-02-20T22:47:39.991Z 13 260-279 140 144 144 245 248 248 F F F <---
156142024-02-20T22:47:39.991Z 14 280-299 136 136 136 241 241 241 F F F
156152024-02-20T22:47:39.991Z 15 300-319 140 144 144 245 248 248 F F F <---
156162024-02-20T22:47:39.991Z 16 320-339 138 138 138 243 243 243 F F F
156172024-02-20T22:47:39.991Z 17 340-359 136 136 136 241 241 241 F F F
156182024-02-20T22:47:39.991Z 18 360-379 140 140 140 245 245 245 F F F
156192024-02-20T22:47:39.991Z 19 380-399 140 140 140 245 245 245 F F F
156202024-02-20T22:47:39.991Z 20 400-419 138 138 138 242 242 242 F T T <---
156212024-02-20T22:47:39.991Z 21 420-439 138 138 138 242 242 242 F T T <---
156222024-02-20T22:47:39.991Z 22 440-459 140 140 140 245 245 245 F T T <---
156232024-02-20T22:47:39.991Z 23 460-479 141 141 141 246 246 246 F F F
156242024-02-20T22:47:39.991Z 24 480-499 141 141 141 246 246 246 F F F
156252024-02-20T22:47:39.991Z 25 500-519 140 144 144 245 248 248 F F F <---
156262024-02-20T22:47:39.991Z 26 520-539 144 144 144 247 248 248 F F F <---
156272024-02-20T22:47:39.991Z 27 540-559 140 144 144 245 248 248 F F F <---
156282024-02-20T22:47:39.991Z 28 560-579 144 144 144 247 247 247 F T T <---
156292024-02-20T22:47:39.991Z 29 580-599 136 136 136 241 241 241 F T T <---
156302024-02-20T22:47:39.991ZMax gen: 144, Max flush: 248
156312024-02-20T22:47:39.991ZError: Difference in extent metadata found!
156322024-02-20T22:47:39.991ZA Difference in extent metadata is expected here
156332024-02-20T22:47:39.991ZOn loop 72
156342024-02-20T22:47:39.991Z
156352024-02-20T22:47:39.995ZVerifying data now
156362024-02-20T22:47:44.755ZNo change after verify
156372024-02-20T22:47:44.755ZLoop: 72 Downstairs dump after verify (and repair):
156382024-02-20T22:47:44.772ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156392024-02-20T22:47:44.772Z 0 000-019 141 141 141 246 246 246 F F F
156402024-02-20T22:47:44.772Z 1 020-039 145 145 145 249 249 249 F F F
156412024-02-20T22:47:44.772Z 2 040-059 141 141 141 246 246 246 F F F
156422024-02-20T22:47:44.772Z 3 060-079 141 141 141 246 246 246 F F F
156432024-02-20T22:47:44.772Z 4 080-099 140 140 140 245 245 245 F F F
156442024-02-20T22:47:44.772Z 5 100-119 132 132 132 234 234 234 F F F
156452024-02-20T22:47:44.772Z 6 120-139 141 141 141 246 246 246 F F F
156462024-02-20T22:47:44.772Z 7 140-159 141 141 141 246 246 246 F F F
156472024-02-20T22:47:44.773Z 8 160-179 141 141 141 246 246 246 F F F
156482024-02-20T22:47:44.773Z 9 180-199 144 144 144 248 248 248 F F F
156492024-02-20T22:47:44.773Z 10 200-219 145 145 145 249 249 249 F F F
156502024-02-20T22:47:44.773Z 11 220-239 145 145 145 249 249 249 F F F
156512024-02-20T22:47:44.773Z 12 240-259 144 144 144 248 248 248 F F F
156522024-02-20T22:47:44.773Z 13 260-279 144 144 144 248 248 248 F F F
156532024-02-20T22:47:44.773Z 14 280-299 136 136 136 241 241 241 F F F
156542024-02-20T22:47:44.773Z 15 300-319 144 144 144 248 248 248 F F F
156552024-02-20T22:47:44.773Z 16 320-339 138 138 138 243 243 243 F F F
156562024-02-20T22:47:44.773Z 17 340-359 136 136 136 241 241 241 F F F
156572024-02-20T22:47:44.773Z 18 360-379 140 140 140 245 245 245 F F F
156582024-02-20T22:47:44.773Z 19 380-399 140 140 140 245 245 245 F F F
156592024-02-20T22:47:44.773Z 20 400-419 145 145 145 249 249 249 F F F
156602024-02-20T22:47:44.773Z 21 420-439 145 145 145 249 249 249 F F F
156612024-02-20T22:47:44.773Z 22 440-459 145 145 145 249 249 249 F F F
156622024-02-20T22:47:44.773Z 23 460-479 141 141 141 246 246 246 F F F
156632024-02-20T22:47:44.773Z 24 480-499 141 141 141 246 246 246 F F F
156642024-02-20T22:47:44.773Z 25 500-519 144 144 144 248 248 248 F F F
156652024-02-20T22:47:44.773Z 26 520-539 144 144 144 248 248 248 F F F
156662024-02-20T22:47:44.773Z 27 540-559 144 144 144 248 248 248 F F F
156672024-02-20T22:47:44.773Z 28 560-579 145 145 145 249 249 249 F F F
156682024-02-20T22:47:44.773Z 29 580-599 145 145 145 249 249 249 F F F
156692024-02-20T22:47:44.773ZMax gen: 145, Max flush: 249
156702024-02-20T22:47:44.773Z
156712024-02-20T22:47:44.774ZBegin loop 73
156722024-02-20T22:47:44.774ZDownstairs to restart: 0
156732024-02-20T22:47:44.774Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25130015Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722}
156742024-02-20T22:47:44.774Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25157992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722}
156752024-02-20T22:47:44.774Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251610819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722}
156762024-02-20T22:47:44.774Z{"msg":"Crucible d21feb14-c097-420d-a2da-7e8dd56d80f6 has session id: 41a81180-e2d9-47be-910d-5500280d8d24","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251633888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156772024-02-20T22:47:44.774ZCrucible runtime is spawned
156782024-02-20T22:47:44.774Z{"msg":"Upstairs opts: Upstairs UUID: d21feb14-c097-420d-a2da-7e8dd56d80f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251686876Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156792024-02-20T22:47:44.774Z{"msg":"Crucible stats registered with UUID: d21feb14-c097-420d-a2da-7e8dd56d80f6","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251721735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156802024-02-20T22:47:44.774Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251746834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722}
156812024-02-20T22:47:44.774Z{"msg":"d21feb14-c097-420d-a2da-7e8dd56d80f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25187298Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156822024-02-20T22:47:44.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251905148Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156832024-02-20T22:47:44.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251921528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156842024-02-20T22:47:44.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251935807Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156852024-02-20T22:47:44.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251968746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156862024-02-20T22:47:44.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.251995325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156872024-02-20T22:47:44.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252017635Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156882024-02-20T22:47:44.774Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252040874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156892024-02-20T22:47:44.774Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252061413Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156902024-02-20T22:47:44.774Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252086352Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156912024-02-20T22:47:44.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252256326Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156922024-02-20T22:47:44.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252296185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156932024-02-20T22:47:44.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.252332893Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156942024-02-20T22:47:44.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253009649Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156952024-02-20T22:47:44.775Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253057927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156962024-02-20T22:47:44.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253086886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156972024-02-20T22:47:44.775Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253128415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156982024-02-20T22:47:44.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253162394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
156992024-02-20T22:47:44.775Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253190493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157002024-02-20T22:47:44.775Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253655456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157012024-02-20T22:47:44.775Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253709414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157022024-02-20T22:47:44.775Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.253839349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157032024-02-20T22:47:44.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254217006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157042024-02-20T22:47:44.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254269144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157052024-02-20T22:47:44.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25437443Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157062024-02-20T22:47:44.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254429088Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157072024-02-20T22:47:44.775Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254484886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157082024-02-20T22:47:44.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254521885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157092024-02-20T22:47:44.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254544794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157102024-02-20T22:47:44.775Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254560013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157112024-02-20T22:47:44.775Z{"msg":"[0]R flush_numbers[0..12]: [246, 249, 246, 246, 245, 234, 246, 246, 246, 248, 249, 249]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
157122024-02-20T22:47:44.775Z,"time":"2024-02-20T22:47:44.254584743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157132024-02-20T22:47:44.775Z{"msg":"[0]R generation[0..12]: [141, 145, 141, 141, 140, 132, 141, 141, 141, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254621591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157142024-02-20T22:47:44.775Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25465934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157152024-02-20T22:47:44.775Z{"msg":"[1]R flush_numbers[0..12]: [246, 249, 246, 246, 245, 234, 246, 246, 246, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254700019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157162024-02-20T22:47:44.775Z{"msg":"[1]R generation[0..12]: [141, 145, 141, 141, 140, 132, 141, 141, 141, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254738387Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157172024-02-20T22:47:44.775Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254778806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157182024-02-20T22:47:44.775Z{"msg":"[2]R flush_numbers[0..12]: [246, 249, 246, 246, 245, 234, 246, 246, 246, 248, 249, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254818174Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157192024-02-20T22:47:44.775Z{"msg":"[2]R generation[0..12]: [141, 145, 141, 141, 140, 132, 141, 141, 141, 144, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254857603Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157202024-02-20T22:47:44.776Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254897861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157212024-02-20T22:47:44.776Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25493745Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157222024-02-20T22:47:44.776Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.254976459Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157232024-02-20T22:47:44.776Z{"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255013547Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157242024-02-20T22:47:44.776Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255049526Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157252024-02-20T22:47:44.776Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157262024-02-20T22:47:44.776Z,"time":"2024-02-20T22:47:44.255087055Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157272024-02-20T22:47:44.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255118433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157282024-02-20T22:47:44.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255134403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157292024-02-20T22:47:44.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255155792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157302024-02-20T22:47:44.776Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255169832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157312024-02-20T22:47:44.776Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255190031Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157322024-02-20T22:47:44.776Z{"msg":"d21feb14-c097-420d-a2da-7e8dd56d80f6 is now active with session: 41a81180-e2d9-47be-910d-5500280d8d24","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25520343Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157332024-02-20T22:47:44.776Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.25522203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722,"session_id":"41a81180-e2d9-47be-910d-5500280d8d24"}
157342024-02-20T22:47:44.776Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:44.255234679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722}
157352024-02-20T22:47:44.776ZLoading write count information from file "/tmp/test_repair_verify.data"
157362024-02-20T22:47:44.776ZRun Repair workload
157372024-02-20T22:47:44.776Z01/30 Write block 283 len 16384 data: 10 10 10 10
157382024-02-20T22:47:44.783Z02/30 Read block 199 len 28672
157392024-02-20T22:47:44.783Z03/30 Read block 278 len 20480
157402024-02-20T22:47:44.783Z04/30 Read block 477 len 16384
157412024-02-20T22:47:44.783Z05/30 Write block 226 len 32768 data: 9 10 13 17 21 19 19 21
157422024-02-20T22:47:44.783Z06/30 Read block 310 len 36864
157432024-02-20T22:47:44.783Z07/30 Write block 118 len 8192 data: 7 11
157442024-02-20T22:47:44.783Z08/30 Write block 103 len 36864 data: 7 9 8 8 8 11 11 13 13
157452024-02-20T22:47:44.783Z09/30 Read block 88 len 20480
157462024-02-20T22:47:44.784Z10/30 Read block 432 len 24576
157472024-02-20T22:47:44.784Z11/30 Read block 227 len 12288
157482024-02-20T22:47:44.784Z12/30 Flush
157492024-02-20T22:47:44.784Z13/30 Write block 88 len 40960 data: 7 8 11 11 8 8 11 10 7 9
157502024-02-20T22:47:44.784Z14/30 Write block 232 len 16384 data: 20 22 21 17
157512024-02-20T22:47:44.784Z15/30 Read block 266 len 40960
157522024-02-20T22:47:44.784Z16/30 Flush
157532024-02-20T22:47:44.784Z17/30 Write block 167 len 12288 data: 8 9 8
157542024-02-20T22:47:44.784Z18/30 Read block 129 len 28672
157552024-02-20T22:47:44.784Z19/30 Write block 92 len 8192 data: 9 9
157562024-02-20T22:47:44.784Z20/30 Read block 92 len 24576
157572024-02-20T22:47:44.784Z21/30 Write block 190 len 40960 data: 9 9 9 8 8 11 9 11 10 10
157582024-02-20T22:47:44.784Z22/30 Write block 97 len 36864 data: 10 6 11 10 10 9 8 10 9
157592024-02-20T22:47:44.784Z23/30 Write block 129 len 12288 data: 10 10 9
157602024-02-20T22:47:44.784Z24/30 Read block 123 len 20480
157612024-02-20T22:47:44.784Z25/30 Write block 179 len 24576 data: 14 14 17 15 13 14
157622024-02-20T22:47:44.784Z26/30 Write block 412 len 16384 data: 15 18 19 17
157632024-02-20T22:47:44.784Z27/30 Write block 500 len 28672 data: 8 7 6 5 8 10 9
157642024-02-20T22:47:44.784Z28/30 Read block 341 len 12288
157652024-02-20T22:47:44.784Z29/30 Write block 37 len 8192 data: 16 15
157662024-02-20T22:47:44.784Z30/30 Write block 374 len 16384 data: 11 10 11 14
157672024-02-20T22:47:44.784Z----------------------------------------------------------------
157682024-02-20T22:47:44.784Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157692024-02-20T22:47:44.784ZStates: Active Active Active
157702024-02-20T22:47:44.784ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157712024-02-20T22:47:44.785Z 1 Acked 1000 Write 4 Done Done Done false
157722024-02-20T22:47:44.785Z 2 Acked 1001 Read 7 Done Done Done false
157732024-02-20T22:47:44.785Z 3 Acked 1002 Read 5 Done Done Done false
157742024-02-20T22:47:44.785Z 4 Acked 1003 Read 4 Done Done Done false
157752024-02-20T22:47:44.785Z 5 Acked 1004 Write 8 Sent Done Done false
157762024-02-20T22:47:44.785Z 6 Acked 1005 Read 9 Sent Done Done false
157772024-02-20T22:47:44.785Z 7 Acked 1006 Write 2 Sent Done Done false
157782024-02-20T22:47:44.785Z 8 Acked 1007 Write 9 Sent Done Done false
157792024-02-20T22:47:44.785Z 9 Acked 1008 Read 5 Sent Done Done false
157802024-02-20T22:47:44.785Z 10 Acked 1009 Read 6 Sent Done Done false
157812024-02-20T22:47:44.785Z 11 Acked 1010 Read 3 Sent Done Done false
157822024-02-20T22:47:44.785Z 12 Acked 1011 Flush 0 Sent Done Done false
157832024-02-20T22:47:44.785Z 13 Acked 1012 Write 10 Sent Done Done false
157842024-02-20T22:47:44.785Z 14 Acked 1013 Write 4 Sent Done Done false
157852024-02-20T22:47:44.785Z 15 Acked 1014 Read 10 Sent Done Done false
157862024-02-20T22:47:44.785Z 16 Acked 1015 Flush 0 Sent Done Done false
157872024-02-20T22:47:44.785Z 17 Acked 1016 Write 3 Sent Done Done false
157882024-02-20T22:47:44.785Z 18 Acked 1017 Read 7 Sent Done Done false
157892024-02-20T22:47:44.785Z 19 Acked 1018 Write 2 Sent Done Done false
157902024-02-20T22:47:44.785Z 20 Acked 1019 Read 6 Sent Done Done false
157912024-02-20T22:47:44.785Z 21 Acked 1020 Write 10 Sent Done Done false
157922024-02-20T22:47:44.785Z 22 Acked 1021 Write 9 Sent Done Done false
157932024-02-20T22:47:44.785Z 23 Acked 1022 Write 3 Sent Done Done false
157942024-02-20T22:47:44.785Z 24 Acked 1023 Read 5 Sent Done Done false
157952024-02-20T22:47:44.785Z 25 Acked 1024 Write 6 Sent Done Done false
157962024-02-20T22:47:44.786Z 26 Acked 1025 Write 4 Sent Done Done false
157972024-02-20T22:47:44.786Z 27 Acked 1026 Write 7 Sent Done Done false
157982024-02-20T22:47:44.786Z 28 Acked 1027 Read 3 Sent Done Done false
157992024-02-20T22:47:44.786Z 29 Acked 1028 Write 2 Sent Done Done false
158002024-02-20T22:47:44.786Z 30 Acked 1029 Write 4 Sent Sent Sent false
158012024-02-20T22:47:44.786Z STATES DS:0 DS:1 DS:2 TOTAL
158022024-02-20T22:47:44.786Z New 0 0 0 0
158032024-02-20T22:47:44.786Z Sent 26 1 1 28
158042024-02-20T22:47:44.786Z Done 4 29 29 62
158052024-02-20T22:47:44.786Z Skipped 0 0 0 0
158062024-02-20T22:47:44.786Z Error 0 0 0 0
158072024-02-20T22:47:44.786ZLast Flush: 0 1015 1015
158082024-02-20T22:47:44.786ZDownstairs last five completed:
158092024-02-20T22:47:44.786ZUpstairs last five completed: 30 29 28 27 26
158102024-02-20T22:47:44.786Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:44.285810415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1722}
158112024-02-20T22:47:44.786ZWrote out file "/tmp/test_repair_verify.data"
158122024-02-20T22:47:44.786Z
158132024-02-20T22:47:44.786ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158142024-02-20T22:47:44.798ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158152024-02-20T22:47:44.798Z 0 000-019 141 141 141 246 246 246 F F F
158162024-02-20T22:47:44.798Z 1 020-039 145 145 145 249 249 249 F T T <---
158172024-02-20T22:47:44.798Z 2 040-059 141 141 141 246 246 246 F F F
158182024-02-20T22:47:44.798Z 3 060-079 141 141 141 246 246 246 F F F
158192024-02-20T22:47:44.798Z 4 080-099 140 146 146 245 251 251 F T T <---
158202024-02-20T22:47:44.798Z 5 100-119 132 146 146 234 250 250 F T T <---
158212024-02-20T22:47:44.798Z 6 120-139 141 141 141 246 246 246 F T T <---
158222024-02-20T22:47:44.798Z 7 140-159 141 141 141 246 246 246 F F F
158232024-02-20T22:47:44.798Z 8 160-179 141 141 141 246 246 246 F T T <---
158242024-02-20T22:47:44.803Z 9 180-199 144 144 144 248 248 248 F T T <---
158252024-02-20T22:47:44.803Z 10 200-219 145 145 145 249 249 249 F F F
158262024-02-20T22:47:44.803Z 11 220-239 145 146 146 249 251 251 F F F <---
158272024-02-20T22:47:44.803Z 12 240-259 144 144 144 248 248 248 F F F
158282024-02-20T22:47:44.803Z 13 260-279 144 144 144 248 248 248 F F F
158292024-02-20T22:47:44.803Z 14 280-299 136 146 146 241 250 250 T F F <---
158302024-02-20T22:47:44.803Z 15 300-319 144 144 144 248 248 248 F F F
158312024-02-20T22:47:44.803Z 16 320-339 138 138 138 243 243 243 F F F
158322024-02-20T22:47:44.803Z 17 340-359 136 136 136 241 241 241 F F F
158332024-02-20T22:47:44.803Z 18 360-379 140 140 140 245 245 245 F T T <---
158342024-02-20T22:47:44.803Z 19 380-399 140 140 140 245 245 245 F F F
158352024-02-20T22:47:44.803Z 20 400-419 145 145 145 249 249 249 F T T <---
158362024-02-20T22:47:44.803Z 21 420-439 145 145 145 249 249 249 F F F
158372024-02-20T22:47:44.804Z 22 440-459 145 145 145 249 249 249 F F F
158382024-02-20T22:47:44.804Z 23 460-479 141 141 141 246 246 246 F F F
158392024-02-20T22:47:44.804Z 24 480-499 141 141 141 246 246 246 F F F
158402024-02-20T22:47:44.804Z 25 500-519 144 144 144 248 248 248 F T T <---
158412024-02-20T22:47:44.804Z 26 520-539 144 144 144 248 248 248 F F F
158422024-02-20T22:47:44.804Z 27 540-559 144 144 144 248 248 248 F F F
158432024-02-20T22:47:44.804Z 28 560-579 145 145 145 249 249 249 F F F
158442024-02-20T22:47:44.804Z 29 580-599 145 145 145 249 249 249 F F F
158452024-02-20T22:47:44.804ZMax gen: 146, Max flush: 251
158462024-02-20T22:47:44.804ZError: Difference in extent metadata found!
158472024-02-20T22:47:44.804ZA Difference in extent metadata is expected here
158482024-02-20T22:47:44.804ZOn loop 73
158492024-02-20T22:47:44.804Z
158502024-02-20T22:47:44.809ZVerifying data now
158512024-02-20T22:47:49.378ZNo change after verify
158522024-02-20T22:47:49.378ZLoop: 73 Downstairs dump after verify (and repair):
158532024-02-20T22:47:49.393ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158542024-02-20T22:47:49.393Z 0 000-019 141 141 141 246 246 246 F F F
158552024-02-20T22:47:49.393Z 1 020-039 147 147 147 252 252 252 F F F
158562024-02-20T22:47:49.393Z 2 040-059 141 141 141 246 246 246 F F F
158572024-02-20T22:47:49.393Z 3 060-079 141 141 141 246 246 246 F F F
158582024-02-20T22:47:49.393Z 4 080-099 147 147 147 252 252 252 F F F
158592024-02-20T22:47:49.393Z 5 100-119 147 147 147 252 252 252 F F F
158602024-02-20T22:47:49.393Z 6 120-139 147 147 147 252 252 252 F F F
158612024-02-20T22:47:49.393Z 7 140-159 141 141 141 246 246 246 F F F
158622024-02-20T22:47:49.393Z 8 160-179 147 147 147 252 252 252 F F F
158632024-02-20T22:47:49.393Z 9 180-199 147 147 147 252 252 252 F F F
158642024-02-20T22:47:49.393Z 10 200-219 145 145 145 249 249 249 F F F
158652024-02-20T22:47:49.398Z 11 220-239 146 146 146 251 251 251 F F F
158662024-02-20T22:47:49.398Z 12 240-259 144 144 144 248 248 248 F F F
158672024-02-20T22:47:49.398Z 13 260-279 144 144 144 248 248 248 F F F
158682024-02-20T22:47:49.398Z 14 280-299 146 146 146 250 250 250 F F F
158692024-02-20T22:47:49.398Z 15 300-319 144 144 144 248 248 248 F F F
158702024-02-20T22:47:49.398Z 16 320-339 138 138 138 243 243 243 F F F
158712024-02-20T22:47:49.398Z 17 340-359 136 136 136 241 241 241 F F F
158722024-02-20T22:47:49.398Z 18 360-379 147 147 147 252 252 252 F F F
158732024-02-20T22:47:49.398Z 19 380-399 140 140 140 245 245 245 F F F
158742024-02-20T22:47:49.398Z 20 400-419 147 147 147 252 252 252 F F F
158752024-02-20T22:47:49.398Z 21 420-439 145 145 145 249 249 249 F F F
158762024-02-20T22:47:49.398Z 22 440-459 145 145 145 249 249 249 F F F
158772024-02-20T22:47:49.398Z 23 460-479 141 141 141 246 246 246 F F F
158782024-02-20T22:47:49.398Z 24 480-499 141 141 141 246 246 246 F F F
158792024-02-20T22:47:49.398Z 25 500-519 147 147 147 252 252 252 F F F
158802024-02-20T22:47:49.398Z 26 520-539 144 144 144 248 248 248 F F F
158812024-02-20T22:47:49.398Z 27 540-559 144 144 144 248 248 248 F F F
158822024-02-20T22:47:49.399Z 28 560-579 145 145 145 249 249 249 F F F
158832024-02-20T22:47:49.399Z 29 580-599 145 145 145 249 249 249 F F F
158842024-02-20T22:47:49.399ZMax gen: 147, Max flush: 252
158852024-02-20T22:47:49.399Z
158862024-02-20T22:47:49.399ZBegin loop 74
158872024-02-20T22:47:49.399ZDownstairs to restart: 1
158882024-02-20T22:47:49.416Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.927857659Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730}
158892024-02-20T22:47:49.416Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928131959Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730}
158902024-02-20T22:47:49.416Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928156028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730}
158912024-02-20T22:47:49.423Z{"msg":"Crucible 12261bc8-ff11-46c6-ae06-665a53e02a55 has session id: a78c63b5-229e-480e-9566-5abf2c9712e7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928190687Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
158922024-02-20T22:47:49.423Z{"msg":"Upstairs opts: Upstairs UUID: 12261bc8-ff11-46c6-ae06-665a53e02a55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928234415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
158932024-02-20T22:47:49.423Z{"msg":"Crucible stats registered with UUID: 12261bc8-ff11-46c6-ae06-665a53e02a55","v":0,"name":"crucible","level"Crucible runtime is spawned
158942024-02-20T22:47:49.423Z:30,"time":"2024-02-20T22:47:48.928283914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
158952024-02-20T22:47:49.423Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.92837237Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730}
158962024-02-20T22:47:49.424Z{"msg":"12261bc8-ff11-46c6-ae06-665a53e02a55 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928503406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
158972024-02-20T22:47:49.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928539924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
158982024-02-20T22:47:49.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928567543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
158992024-02-20T22:47:49.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928616532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159002024-02-20T22:47:49.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928638231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159012024-02-20T22:47:49.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.92865898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159022024-02-20T22:47:49.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928698399Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159032024-02-20T22:47:49.424Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928721728Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159042024-02-20T22:47:49.424Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928756397Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159052024-02-20T22:47:49.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928783926Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159062024-02-20T22:47:49.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928847953Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159072024-02-20T22:47:49.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.928968289Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159082024-02-20T22:47:49.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.929111884Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159092024-02-20T22:47:49.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.929694383Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159102024-02-20T22:47:49.424Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.929730832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159112024-02-20T22:47:49.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.929748161Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159122024-02-20T22:47:49.424Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.929763261Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159132024-02-20T22:47:49.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.92978408Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159142024-02-20T22:47:49.424Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.929798559Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159152024-02-20T22:47:49.424Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.930404518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159162024-02-20T22:47:49.424Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.930460376Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159172024-02-20T22:47:49.424Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.930578231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159182024-02-20T22:47:49.424Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.93090402Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159192024-02-20T22:47:49.424Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.930942918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159202024-02-20T22:47:49.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.930966898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159212024-02-20T22:47:49.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.930987957Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159222024-02-20T22:47:49.425Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931020806Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159232024-02-20T22:47:49.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931046795Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159242024-02-20T22:47:49.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931215939Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159252024-02-20T22:47:49.425Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931258927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159262024-02-20T22:47:49.425Z{"msg":"[0]R flush_numbers[0..12]: [246, 252, 246, 246, 252, 252, 252, 246, 252, 252, 249, 251]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931277817Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159272024-02-20T22:47:49.425Z{"msg":"[0]R generation[0..12]: [141, 147, 141, 141, 147, 147, 147, 141, 147, 147, 145, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931300916Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159282024-02-20T22:47:49.425Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931316465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159292024-02-20T22:47:49.425Z{"msg":"[1]R flush_numbers[0..12]: [246, 252, 246, 246, 252, 252, 252, 246, 252, 252, 249, 251]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931338824Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159302024-02-20T22:47:49.425Z{"msg":"[1]R generation[0..12]: [141, 147, 141, 141, 147, 147, 147, 141, 147, 147, 145, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931353614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159312024-02-20T22:47:49.425Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931376203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159322024-02-20T22:47:49.425Z{"msg":"[2]R flush_numbers[0..12]: [246, 252, 246, 246, 252, 252, 252, 246, 252, 252, 249, 251]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931390933Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159332024-02-20T22:47:49.425Z{"msg":"[2]R generation[0..12]: [141, 147, 141, 141, 147, 147, 147, 141, 147, 147, 145, 146]","v":Wait for a query_work_queue command to finish before sending IO
159342024-02-20T22:47:49.425Z0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931418532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159352024-02-20T22:47:49.425Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931434181Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159362024-02-20T22:47:49.425Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931448051Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159372024-02-20T22:47:49.425Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.93146188Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159382024-02-20T22:47:49.425Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931474999Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159392024-02-20T22:47:49.425Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931488849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159402024-02-20T22:47:49.425Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931510188Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159412024-02-20T22:47:49.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931522988Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159422024-02-20T22:47:49.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931543477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159432024-02-20T22:47:49.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931557247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159442024-02-20T22:47:49.426Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931578646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159452024-02-20T22:47:49.426Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931591775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159462024-02-20T22:47:49.426Z{"msg":"12261bc8-ff11-46c6-ae06-665a53e02a55 is now active with session: a78c63b5-229e-480e-9566-5abf2c9712e7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931612154Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159472024-02-20T22:47:49.426Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931625414Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730,"session_id":"a78c63b5-229e-480e-9566-5abf2c9712e7"}
159482024-02-20T22:47:49.426Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:48.931644404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730}
159492024-02-20T22:47:49.426ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159502024-02-20T22:47:49.426ZLoading write count information from file "/tmp/test_repair_verify.data"
159512024-02-20T22:47:49.426ZRun Repair workload
159522024-02-20T22:47:49.430Z01/30 Write block 347 len 16384 data: 12 13 10 11
159532024-02-20T22:47:49.430Z02/30 Read block 343 len 8192
159542024-02-20T22:47:49.430Z03/30 Read block 480 len 24576
159552024-02-20T22:47:49.430Z04/30 Write block 293 len 16384 data: 6 5 4 7
159562024-02-20T22:47:49.430Z05/30 Write block 54 len 28672 data: 12 10 11 10 10 13 11
159572024-02-20T22:47:49.430Z06/30 Read block 111 len 24576
159582024-02-20T22:47:49.430Z07/30 Write block 318 len 8192 data: 13 13
159592024-02-20T22:47:49.430Z08/30 Read block 480 len 16384
159602024-02-20T22:47:49.430Z09/30 Flush
159612024-02-20T22:47:49.437Z10/30 Write block 383 len 40960 data: 9 10 8 6 5 7 10 11 10 12
159622024-02-20T22:47:49.437Z11/30 Read block 155 len 12288
159632024-02-20T22:47:49.437Z12/30 Write block 301 len 20480 data: 16 18 18 16 16
159642024-02-20T22:47:49.437Z13/30 Flush
159652024-02-20T22:47:49.440Z14/30 Write block 427 len 4096 data: 7
159662024-02-20T22:47:49.440Z15/30 Write block 116 len 8192 data: 11 12
159672024-02-20T22:47:49.441Z16/30 Write block 511 len 40960 data: 15 10 11 11 11 10 12 10 9 12
159682024-02-20T22:47:49.441Z17/30 Write block 379 len 36864 data: 10 10 9 11 10 11 9 7 6
159692024-02-20T22:47:49.441Z18/30 Read block 113 len 36864
159702024-02-20T22:47:49.441Z19/30 Read block 310 len 12288
159712024-02-20T22:47:49.444Z20/30 Write block 27 len 12288 data: 15 16 20
159722024-02-20T22:47:49.444Z21/30 Read block 489 len 36864
159732024-02-20T22:47:49.444Z22/30 Flush
159742024-02-20T22:47:49.447Z23/30 Write block 216 len 20480 data: 8 5 6 6 8
159752024-02-20T22:47:49.447Z24/30 Read block 309 len 28672
159762024-02-20T22:47:49.447Z25/30 Write block 177 len 12288 data: 12 12 15
159772024-02-20T22:47:49.451Z26/30 Write block 351 len 8192 data: 9 7
159782024-02-20T22:47:49.451Z27/30 Read block 69 len 16384
159792024-02-20T22:47:49.451Z28/30 Read block 297 len 36864
159802024-02-20T22:47:49.451Z29/30 Read block 453 len 24576
159812024-02-20T22:47:49.455Z30/30 Write block 485 len 40960 data: 14 12 11 11 9 9 10 12 14 13
159822024-02-20T22:47:49.455Z----------------------------------------------------------------
159832024-02-20T22:47:49.456Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159842024-02-20T22:47:49.456ZStates: Active Active Active
159852024-02-20T22:47:49.456ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159862024-02-20T22:47:49.456Z 1 Acked 1000 Write 4 Done Done Done false
159872024-02-20T22:47:49.456Z 2 Acked 1001 Read 2 Done Done Done false
159882024-02-20T22:47:49.456Z 3 Acked 1002 Read 6 Done Sent Done false
159892024-02-20T22:47:49.456Z 4 Acked 1003 Write 4 Done Sent Done false
159902024-02-20T22:47:49.456Z 5 Acked 1004 Write 7 Done Sent Done false
159912024-02-20T22:47:49.456Z 6 Acked 1005 Read 6 Done Sent Done false
159922024-02-20T22:47:49.456Z 7 Acked 1006 Write 2 Done Sent Done false
159932024-02-20T22:47:49.456Z 8 Acked 1007 Read 4 Done Sent Done false
159942024-02-20T22:47:49.456Z 9 Acked 1008 Flush 0 Done Sent Done false
159952024-02-20T22:47:49.456Z 10 Acked 1009 Write 10 Done Sent Done false
159962024-02-20T22:47:49.456Z 11 Acked 1010 Read 3 Done Sent Done false
159972024-02-20T22:47:49.456Z 12 Acked 1011 Write 5 Done Sent Done false
159982024-02-20T22:47:49.456Z 13 Acked 1012 Flush 0 Done Sent Done false
159992024-02-20T22:47:49.456Z 14 Acked 1013 Write 1 Done Sent Done false
160002024-02-20T22:47:49.456Z 15 Acked 1014 Write 2 Done Sent Done false
160012024-02-20T22:47:49.456Z 16 Acked 1015 Write 10 Done Sent Done false
160022024-02-20T22:47:49.456Z 17 Acked 1016 Write 9 Done Sent Done false
160032024-02-20T22:47:49.456Z 18 Acked 1017 Read 9 Done Sent Done false
160042024-02-20T22:47:49.456Z 19 Acked 1018 Read 3 Done Sent Done false
160052024-02-20T22:47:49.456Z 20 Acked 1019 Write 3 Done Sent Done false
160062024-02-20T22:47:49.456Z 21 Acked 1020 Read 9 Done Sent Done false
160072024-02-20T22:47:49.457Z 22 Acked 1021 Flush 0 Done Sent Done false
160082024-02-20T22:47:49.457Z 23 Acked 1022 Write 5 Done Sent Done false
160092024-02-20T22:47:49.457Z 24 Acked 1023 Read 7 Done Sent Done false
160102024-02-20T22:47:49.457Z 25 Acked 1024 Write 3 Done Sent Done false
160112024-02-20T22:47:49.457Z 26 Acked 1025 Write 2 Done Sent Done false
160122024-02-20T22:47:49.457Z 27 Acked 1026 Read 4 Done Sent Done false
160132024-02-20T22:47:49.457Z 28 Acked 1027 Read 9 Done Sent Done false
160142024-02-20T22:47:49.457Z 29 Acked 1028 Read 6 Done Sent Done false
160152024-02-20T22:47:49.457Z 30 Acked 1029 Write 10 Sent Sent Sent false
160162024-02-20T22:47:49.457Z STATES DS:0 DS:1 DS:2 TOTAL
160172024-02-20T22:47:49.457Z New 0 0 0 0
160182024-02-20T22:47:49.457Z Sent 1 28 1 30
160192024-02-20T22:47:49.457Z Done 29 2 29 60
160202024-02-20T22:47:49.457Z Skipped 0 0 0 0
160212024-02-20T22:47:49.457Z Error 0 0 0 0
160222024-02-20T22:47:49.457ZLast Flush: 1021 0 1021
160232024-02-20T22:47:49.457ZDownstairs last five completed:
160242024-02-20T22:47:49.457ZUpstairs last five completed: 30 29 28 27 26
160252024-02-20T22:47:49.457Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:48.964699341Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1730}
160262024-02-20T22:47:49.457ZWrote out file "/tmp/test_repair_verify.data"
160272024-02-20T22:47:49.461Z
160282024-02-20T22:47:49.461ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160292024-02-20T22:47:49.476ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160302024-02-20T22:47:49.477Z 0 000-019 141 141 141 246 246 246 F F F
160312024-02-20T22:47:49.477Z 1 020-039 148 147 148 255 252 255 F F F <---
160322024-02-20T22:47:49.477Z 2 040-059 148 141 148 253 246 253 F F F <---
160332024-02-20T22:47:49.477Z 3 060-079 148 141 148 253 246 253 F F F <---
160342024-02-20T22:47:49.477Z 4 080-099 147 147 147 252 252 252 F F F
160352024-02-20T22:47:49.477Z 5 100-119 148 147 148 255 252 255 F F F <---
160362024-02-20T22:47:49.477Z 6 120-139 147 147 147 252 252 252 F F F
160372024-02-20T22:47:49.477Z 7 140-159 141 141 141 246 246 246 F F F
160382024-02-20T22:47:49.477Z 8 160-179 147 147 147 252 252 252 T F T <---
160392024-02-20T22:47:49.477Z 9 180-199 147 147 147 252 252 252 F F F
160402024-02-20T22:47:49.477Z 10 200-219 145 145 145 249 249 249 T F T <---
160412024-02-20T22:47:49.477Z 11 220-239 146 146 146 251 251 251 T F T <---
160422024-02-20T22:47:49.477Z 12 240-259 144 144 144 248 248 248 F F F
160432024-02-20T22:47:49.481Z 13 260-279 144 144 144 248 248 248 F F F
160442024-02-20T22:47:49.481Z 14 280-299 148 146 148 253 250 253 F F F <---
160452024-02-20T22:47:49.481Z 15 300-319 148 144 148 254 248 254 F F F <---
160462024-02-20T22:47:49.481Z 16 320-339 138 138 138 243 243 243 F F F
160472024-02-20T22:47:49.481Z 17 340-359 148 136 148 253 241 253 T T T <---
160482024-02-20T22:47:49.481Z 18 360-379 148 147 148 255 252 255 F F F <---
160492024-02-20T22:47:49.481Z 19 380-399 148 140 148 255 245 255 F F F <---
160502024-02-20T22:47:49.481Z 20 400-419 147 147 147 252 252 252 F F F
160512024-02-20T22:47:49.481Z 21 420-439 148 145 148 255 249 255 F F F <---
160522024-02-20T22:47:49.481Z 22 440-459 145 145 145 249 249 249 F F F
160532024-02-20T22:47:49.481Z 23 460-479 141 141 141 246 246 246 F F F
160542024-02-20T22:47:49.481Z 24 480-499 141 141 141 246 246 246 T F T <---
160552024-02-20T22:47:49.481Z 25 500-519 148 147 148 255 252 255 F F F <---
160562024-02-20T22:47:49.481Z 26 520-539 148 144 148 255 248 255 F F F <---
160572024-02-20T22:47:49.481Z 27 540-559 144 144 144 248 248 248 F F F
160582024-02-20T22:47:49.481Z 28 560-579 145 145 145 249 249 249 F F F
160592024-02-20T22:47:49.481Z 29 580-599 145 145 145 249 249 249 F F F
160602024-02-20T22:47:49.481ZMax gen: 148, Max flush: 255
160612024-02-20T22:47:49.481ZError: Difference in extent metadata found!
160622024-02-20T22:47:49.482ZA Difference in extent metadata is expected here
160632024-02-20T22:47:49.482ZOn loop 74
160642024-02-20T22:47:49.482Z
160652024-02-20T22:47:49.485ZVerifying data now
160662024-02-20T22:47:54.248ZNo change after verify
160672024-02-20T22:47:54.248ZLoop: 74 Downstairs dump after verify (and repair):
160682024-02-20T22:47:54.262ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160692024-02-20T22:47:54.262Z 0 000-019 141 141 141 246 246 246 F F F
160702024-02-20T22:47:54.262Z 1 020-039 148 148 148 255 255 255 F F F
160712024-02-20T22:47:54.262Z 2 040-059 148 148 148 253 253 253 F F F
160722024-02-20T22:47:54.262Z 3 060-079 148 148 148 253 253 253 F F F
160732024-02-20T22:47:54.263Z 4 080-099 147 147 147 252 252 252 F F F
160742024-02-20T22:47:54.263Z 5 100-119 148 148 148 255 255 255 F F F
160752024-02-20T22:47:54.263Z 6 120-139 147 147 147 252 252 252 F F F
160762024-02-20T22:47:54.263Z 7 140-159 141 141 141 246 246 246 F F F
160772024-02-20T22:47:54.263Z 8 160-179 149 149 149 256 256 256 F F F
160782024-02-20T22:47:54.263Z 9 180-199 147 147 147 252 252 252 F F F
160792024-02-20T22:47:54.267Z 10 200-219 149 149 149 256 256 256 F F F
160802024-02-20T22:47:54.267Z 11 220-239 149 149 149 256 256 256 F F F
160812024-02-20T22:47:54.267Z 12 240-259 144 144 144 248 248 248 F F F
160822024-02-20T22:47:54.267Z 13 260-279 144 144 144 248 248 248 F F F
160832024-02-20T22:47:54.267Z 14 280-299 148 148 148 253 253 253 F F F
160842024-02-20T22:47:54.267Z 15 300-319 148 148 148 254 254 254 F F F
160852024-02-20T22:47:54.267Z 16 320-339 138 138 138 243 243 243 F F F
160862024-02-20T22:47:54.267Z 17 340-359 149 149 149 256 256 256 F F F
160872024-02-20T22:47:54.267Z 18 360-379 148 148 148 255 255 255 F F F
160882024-02-20T22:47:54.267Z 19 380-399 148 148 148 255 255 255 F F F
160892024-02-20T22:47:54.267Z 20 400-419 147 147 147 252 252 252 F F F
160902024-02-20T22:47:54.267Z 21 420-439 148 148 148 255 255 255 F F F
160912024-02-20T22:47:54.267Z 22 440-459 145 145 145 249 249 249 F F F
160922024-02-20T22:47:54.267Z 23 460-479 141 141 141 246 246 246 F F F
160932024-02-20T22:47:54.267Z 24 480-499 149 149 149 256 256 256 F F F
160942024-02-20T22:47:54.267Z 25 500-519 148 148 148 255 255 255 F F F
160952024-02-20T22:47:54.267Z 26 520-539 148 148 148 255 255 255 F F F
160962024-02-20T22:47:54.267Z 27 540-559 144 144 144 248 248 248 F F F
160972024-02-20T22:47:54.267Z 28 560-579 145 145 145 249 249 249 F F F
160982024-02-20T22:47:54.267Z 29 580-599 145 145 145 249 249 249 F F F
160992024-02-20T22:47:54.267ZMax gen: 149, Max flush: 256
161002024-02-20T22:47:54.268Z
161012024-02-20T22:47:54.268ZBegin loop 75
161022024-02-20T22:47:54.268ZDownstairs to restart: 0
161032024-02-20T22:47:54.284Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.795624466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738}
161042024-02-20T22:47:54.284Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
161052024-02-20T22:47:54.291Z,"time":"2024-02-20T22:47:53.795940235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738}
161062024-02-20T22:47:54.291Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.795986144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738}
161072024-02-20T22:47:54.291Z{"msg":"Crucible 15cf09a3-edfa-4c09-ba49-161541010848 has session id: 75453223-be33-4d27-bd3d-6e5f4ec304fd","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796017072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161082024-02-20T22:47:54.291Z{"msg":"Upstairs opts: Upstairs UUID: 15cf09a3-edfa-4c09-ba49-161541010848, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796069401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161092024-02-20T22:47:54.291Z{"msg":"Crucible stats registered with UUID: 15cf09a3-edfa-4c09-ba49-161541010848","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.79610264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161102024-02-20T22:47:54.291Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796140028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738}
161112024-02-20T22:47:54.291Z{"msg":"15cf09a3-edfa-4c09-ba49-161541010848 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796174937Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161122024-02-20T22:47:54.291Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796201256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161132024-02-20T22:47:54.291Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796220855Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161142024-02-20T22:47:54.291Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796238804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161152024-02-20T22:47:54.291Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796259784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161162024-02-20T22:47:54.291Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796302062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161172024-02-20T22:47:54.292Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796335361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161182024-02-20T22:47:54.292Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.7963647Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161192024-02-20T22:47:54.292Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.7966549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161202024-02-20T22:47:54.292Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796698308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161212024-02-20T22:47:54.292Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796740927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161222024-02-20T22:47:54.292Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796775945Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161232024-02-20T22:47:54.292Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.796814874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161242024-02-20T22:47:54.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797330325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161252024-02-20T22:47:54.292Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797380164Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161262024-02-20T22:47:54.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797460501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161272024-02-20T22:47:54.292Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797514099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161282024-02-20T22:47:54.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797558307Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161292024-02-20T22:47:54.292Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797599276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161302024-02-20T22:47:54.292Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797932534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161312024-02-20T22:47:54.292Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.797971502Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161322024-02-20T22:47:54.292Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798004881Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161332024-02-20T22:47:54.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798436866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161342024-02-20T22:47:54.292Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798479544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161352024-02-20T22:47:54.292Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798506443Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161362024-02-20T22:47:54.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798535632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161372024-02-20T22:47:54.292Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798560641Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161382024-02-20T22:47:54.292Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.79858848Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161392024-02-20T22:47:54.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.79861147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161402024-02-20T22:47:54.292Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798640039Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161412024-02-20T22:47:54.292Z{"msg":"[0]R flush_numbers[0..12]: [246, 255, 253, 253, 252, 255, 252, 246, 256, 252, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798665228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161422024-02-20T22:47:54.293Z{"msg":"[0]R generation[0..12]: [141, 148, 148, 148, 147, 148, 147, 141, 149, 147, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798693817Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161432024-02-20T22:47:54.293Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798719986Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161442024-02-20T22:47:54.293Z{"msg":"[1]R flush_numbers[0..12]: [246, 255, 253, 253, 252, 255, 252, 246, 256, 252, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798753025Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
161452024-02-20T22:47:54.293Z75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161462024-02-20T22:47:54.293Z{"msg":"[1]R generation[0..12]: [141, 148, 148, 148, 147, 148, 147, 141, 149, 147, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798783933Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161472024-02-20T22:47:54.293Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798814103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161482024-02-20T22:47:54.293Z{"msg":"[2]R flush_numbers[0..12]: [246, 255, 253, 253, 252, 255, 252, 246, 256, 252, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798840451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161492024-02-20T22:47:54.293Z{"msg":"[2]R generation[0..12]: [141, 148, 148, 148, 147, 148, 147, 141, 149, 147, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.79887175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161502024-02-20T22:47:54.293Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.79889814Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161512024-02-20T22:47:54.293Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798928168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161522024-02-20T22:47:54.293Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798952587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161532024-02-20T22:47:54.293Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.798979006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161542024-02-20T22:47:54.293Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799002336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161552024-02-20T22:47:54.293Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799030455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161562024-02-20T22:47:54.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799052344Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161572024-02-20T22:47:54.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799076393Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161582024-02-20T22:47:54.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799114292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161592024-02-20T22:47:54.293Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799134321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161602024-02-20T22:47:54.293Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.79917167Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161612024-02-20T22:47:54.293Z{"msg":"15cf09a3-edfa-4c09-ba49-161541010848 is now active with session: 75453223-be33-4d27-bd3d-6e5f4ec304fd","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799209648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161622024-02-20T22:47:54.293Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799238237Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738,"session_id":"75453223-be33-4d27-bd3d-6e5f4ec304fd"}
161632024-02-20T22:47:54.293Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:53.799252417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738}
161642024-02-20T22:47:54.293ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161652024-02-20T22:47:54.298ZLoading write count information from file "/tmp/test_repair_verify.data"
161662024-02-20T22:47:54.298ZRun Repair workload
161672024-02-20T22:47:54.298Z01/30 Write block 403 len 20480 data: 14 14 13 13 11
161682024-02-20T22:47:54.298Z02/30 Write block 440 len 36864 data: 13 15 12 9 8 11 11 11 9
161692024-02-20T22:47:54.298Z03/30 Read block 284 len 4096
161702024-02-20T22:47:54.298Z04/30 Write block 502 len 32768 data: 7 6 9 11 10 11 10 10
161712024-02-20T22:47:54.298Z05/30 Write block 322 len 24576 data: 16 12 10 15 16 14
161722024-02-20T22:47:54.298Z06/30 Read block 241 len 16384
161732024-02-20T22:47:54.298Z07/30 Read block 138 len 12288
161742024-02-20T22:47:54.298Z08/30 Write block 565 len 16384 data: 9 9 8 8
161752024-02-20T22:47:54.298Z09/30 Write block 208 len 8192 data: 11 12
161762024-02-20T22:47:54.298Z10/30 Read block 200 len 24576
161772024-02-20T22:47:54.298Z11/30 Read block 194 len 40960
161782024-02-20T22:47:54.302Z12/30 Read block 255 len 24576
161792024-02-20T22:47:54.302Z13/30 Write block 575 len 20480 data: 8 10 14 17 20
161802024-02-20T22:47:54.302Z14/30 Read block 521 len 24576
161812024-02-20T22:47:54.302Z15/30 Read block 7 len 24576
161822024-02-20T22:47:54.302Z16/30 Flush
161832024-02-20T22:47:54.308Z17/30 Write block 34 len 12288 data: 20 21 18
161842024-02-20T22:47:54.309Z18/30 Write block 530 len 24576 data: 11 10 7 8 11 14
161852024-02-20T22:47:54.309Z19/30 Read block 359 len 12288
161862024-02-20T22:47:54.321Z20/30 Write block 275 len 16384 data: 9 7 8 5
161872024-02-20T22:47:54.321Z21/30 Write block 234 len 12288 data: 22 18 17
161882024-02-20T22:47:54.321Z22/30 Flush
161892024-02-20T22:47:54.321Z23/30 Write block 515 len 24576 data: 12 11 13 11 10 13
161902024-02-20T22:47:54.322Z24/30 Write block 355 len 40960 data: 6 9 5 8 13 15 14 17 19 17
161912024-02-20T22:47:54.322Z25/30 Write block 24 len 28672 data: 8 13 11 16 17 21 20
161922024-02-20T22:47:54.322Z26/30 Read block 158 len 12288
161932024-02-20T22:47:54.322Z27/30 Read block 61 len 12288
161942024-02-20T22:47:54.329Z28/30 Read block 233 len 20480
161952024-02-20T22:47:54.329Z29/30 Write block 116 len 20480 data: 12 13 8 12 12
161962024-02-20T22:47:54.329Z30/30 Write block 348 len 12288 data: 14 11 12
161972024-02-20T22:47:54.329Z----------------------------------------------------------------
161982024-02-20T22:47:54.329Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161992024-02-20T22:47:54.329ZStates: Active Active Active
162002024-02-20T22:47:54.329ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162012024-02-20T22:47:54.330Z 1 Acked 1000 Write 5 Done Done Done false
162022024-02-20T22:47:54.330Z 2 Acked 1001 Write 9 Sent Done Done false
162032024-02-20T22:47:54.330Z 3 Acked 1002 Read 1 Sent Done Done false
162042024-02-20T22:47:54.330Z 4 Acked 1003 Write 8 Sent Done Done false
162052024-02-20T22:47:54.330Z 5 Acked 1004 Write 6 Sent Done Done false
162062024-02-20T22:47:54.330Z 6 Acked 1005 Read 4 Sent Done Done false
162072024-02-20T22:47:54.330Z 7 Acked 1006 Read 3 Sent Done Done false
162082024-02-20T22:47:54.330Z 8 Acked 1007 Write 4 Sent Done Done false
162092024-02-20T22:47:54.330Z 9 Acked 1008 Write 2 Sent Done Done false
162102024-02-20T22:47:54.330Z 10 Acked 1009 Read 6 Sent Done Done false
162112024-02-20T22:47:54.330Z 11 Acked 1010 Read 10 Sent Done Done false
162122024-02-20T22:47:54.330Z 12 Acked 1011 Read 6 Sent Done Done false
162132024-02-20T22:47:54.330Z 13 Acked 1012 Write 5 Sent Done Done false
162142024-02-20T22:47:54.330Z 14 Acked 1013 Read 6 Sent Done Done false
162152024-02-20T22:47:54.330Z 15 Acked 1014 Read 6 Sent Done Done false
162162024-02-20T22:47:54.330Z 16 Acked 1015 Flush 0 Sent Done Done false
162172024-02-20T22:47:54.330Z 17 Acked 1016 Write 3 Sent Done Done false
162182024-02-20T22:47:54.330Z 18 Acked 1017 Write 6 Sent Done Done false
162192024-02-20T22:47:54.330Z 19 Acked 1018 Read 3 Sent Done Done false
162202024-02-20T22:47:54.330Z 20 Acked 1019 Write 4 Sent Done Done false
162212024-02-20T22:47:54.330Z 21 Acked 1020 Write 3 Sent Done Done false
162222024-02-20T22:47:54.330Z 22 Acked 1021 Flush 0 Sent Done Done false
162232024-02-20T22:47:54.330Z 23 Acked 1022 Write 6 Sent Done Done false
162242024-02-20T22:47:54.331Z 24 Acked 1023 Write 10 Sent Done Done false
162252024-02-20T22:47:54.331Z 25 Acked 1024 Write 7 Sent Done Done false
162262024-02-20T22:47:54.331Z 26 Acked 1025 Read 3 Sent Done Done false
162272024-02-20T22:47:54.331Z 27 Acked 1026 Read 3 Sent Done Done false
162282024-02-20T22:47:54.331Z 28 Acked 1027 Read 5 Sent Done Done false
162292024-02-20T22:47:54.331Z 29 Acked 1028 Write 5 Sent Sent Sent false
162302024-02-20T22:47:54.331Z 30 Acked 1029 Write 3 Sent Sent Sent false
162312024-02-20T22:47:54.331Z STATES DS:0 DS:1 DS:2 TOTAL
162322024-02-20T22:47:54.331Z New 0 0 0 0
162332024-02-20T22:47:54.331Z Sent 29 2 2 33
162342024-02-20T22:47:54.331Z Done 1 28 28 57
162352024-02-20T22:47:54.331Z Skipped 0 0 0 0
162362024-02-20T22:47:54.331Z Error 0 0 0 0
162372024-02-20T22:47:54.331ZLast Flush: 0 1021 1021
162382024-02-20T22:47:54.331ZDownstairs last five completed:
162392024-02-20T22:47:54.331ZUpstairs last five completed: 30 29 28 27 26
162402024-02-20T22:47:54.331Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:53.829755765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1738}
162412024-02-20T22:47:54.331ZWrote out file "/tmp/test_repair_verify.data"
162422024-02-20T22:47:54.331Z
162432024-02-20T22:47:54.331ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162442024-02-20T22:47:54.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162452024-02-20T22:47:54.342Z 0 000-019 141 141 141 246 246 246 F F F
162462024-02-20T22:47:54.342Z 1 020-039 148 150 150 255 258 258 F T T <---
162472024-02-20T22:47:54.342Z 2 040-059 148 148 148 253 253 253 F F F
162482024-02-20T22:47:54.342Z 3 060-079 148 148 148 253 253 253 F F F
162492024-02-20T22:47:54.342Z 4 080-099 147 147 147 252 252 252 F F F
162502024-02-20T22:47:54.342Z 5 100-119 148 148 148 255 255 255 F T T <---
162512024-02-20T22:47:54.342Z 6 120-139 147 147 147 252 252 252 F T T <---
162522024-02-20T22:47:54.342Z 7 140-159 141 141 141 246 246 246 F F F
162532024-02-20T22:47:54.342Z 8 160-179 149 149 149 256 256 256 F F F
162542024-02-20T22:47:54.342Z 9 180-199 147 147 147 252 252 252 F F F
162552024-02-20T22:47:54.350Z 10 200-219 149 150 150 256 257 257 F F F <---
162562024-02-20T22:47:54.350Z 11 220-239 149 150 150 256 258 258 F F F <---
162572024-02-20T22:47:54.350Z 12 240-259 144 144 144 248 248 248 F F F
162582024-02-20T22:47:54.350Z 13 260-279 144 150 150 248 258 258 F F F <---
162592024-02-20T22:47:54.350Z 14 280-299 148 148 148 253 253 253 F F F
162602024-02-20T22:47:54.350Z 15 300-319 148 148 148 254 254 254 F F F
162612024-02-20T22:47:54.350Z 16 320-339 138 150 150 243 257 257 F F F <---
162622024-02-20T22:47:54.350Z 17 340-359 149 149 149 256 256 256 F T T <---
162632024-02-20T22:47:54.350Z 18 360-379 148 148 148 255 255 255 F T T <---
162642024-02-20T22:47:54.350Z 19 380-399 148 148 148 255 255 255 F F F
162652024-02-20T22:47:54.350Z 20 400-419 147 150 150 252 257 257 T F F <---
162662024-02-20T22:47:54.350Z 21 420-439 148 148 148 255 255 255 F F F
162672024-02-20T22:47:54.350Z 22 440-459 145 150 150 249 257 257 F F F <---
162682024-02-20T22:47:54.350Z 23 460-479 141 141 141 246 246 246 F F F
162692024-02-20T22:47:54.351Z 24 480-499 149 149 149 256 256 256 F F F
162702024-02-20T22:47:54.351Z 25 500-519 148 150 150 255 257 257 F T T <---
162712024-02-20T22:47:54.351Z 26 520-539 148 150 150 255 258 258 F T T <---
162722024-02-20T22:47:54.351Z 27 540-559 144 144 144 248 248 248 F F F
162732024-02-20T22:47:54.351Z 28 560-579 145 150 150 249 257 257 F F F <---
162742024-02-20T22:47:54.351Z 29 580-599 145 145 145 249 249 249 F F F
162752024-02-20T22:47:54.351ZMax gen: 150, Max flush: 258
162762024-02-20T22:47:54.351ZError: Difference in extent metadata found!
162772024-02-20T22:47:54.351ZA Difference in extent metadata is expected here
162782024-02-20T22:47:54.351ZOn loop 75
162792024-02-20T22:47:54.351Z
162802024-02-20T22:47:54.355ZVerifying data now
162812024-02-20T22:47:59.043ZNo change after verify
162822024-02-20T22:47:59.043ZLoop: 75 Downstairs dump after verify (and repair):
162832024-02-20T22:47:59.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162842024-02-20T22:47:59.057Z 0 000-019 141 141 141 246 246 246 F F F
162852024-02-20T22:47:59.057Z 1 020-039 151 151 151 259 259 259 F F F
162862024-02-20T22:47:59.057Z 2 040-059 148 148 148 253 253 253 F F F
162872024-02-20T22:47:59.057Z 3 060-079 148 148 148 253 253 253 F F F
162882024-02-20T22:47:59.057Z 4 080-099 147 147 147 252 252 252 F F F
162892024-02-20T22:47:59.057Z 5 100-119 151 151 151 259 259 259 F F F
162902024-02-20T22:47:59.057Z 6 120-139 151 151 151 259 259 259 F F F
162912024-02-20T22:47:59.057Z 7 140-159 141 141 141 246 246 246 F F F
162922024-02-20T22:47:59.057Z 8 160-179 149 149 149 256 256 256 F F F
162932024-02-20T22:47:59.057Z 9 180-199 147 147 147 252 252 252 F F F
162942024-02-20T22:47:59.061Z 10 200-219 150 150 150 257 257 257 F F F
162952024-02-20T22:47:59.061Z 11 220-239 150 150 150 258 258 258 F F F
162962024-02-20T22:47:59.061Z 12 240-259 144 144 144 248 248 248 F F F
162972024-02-20T22:47:59.062Z 13 260-279 150 150 150 258 258 258 F F F
162982024-02-20T22:47:59.062Z 14 280-299 148 148 148 253 253 253 F F F
162992024-02-20T22:47:59.062Z 15 300-319 148 148 148 254 254 254 F F F
163002024-02-20T22:47:59.062Z 16 320-339 150 150 150 257 257 257 F F F
163012024-02-20T22:47:59.062Z 17 340-359 151 151 151 259 259 259 F F F
163022024-02-20T22:47:59.062Z 18 360-379 151 151 151 259 259 259 F F F
163032024-02-20T22:47:59.062Z 19 380-399 148 148 148 255 255 255 F F F
163042024-02-20T22:47:59.062Z 20 400-419 150 150 150 257 257 257 F F F
163052024-02-20T22:47:59.062Z 21 420-439 148 148 148 255 255 255 F F F
163062024-02-20T22:47:59.062Z 22 440-459 150 150 150 257 257 257 F F F
163072024-02-20T22:47:59.062Z 23 460-479 141 141 141 246 246 246 F F F
163082024-02-20T22:47:59.062Z 24 480-499 149 149 149 256 256 256 F F F
163092024-02-20T22:47:59.062Z 25 500-519 151 151 151 259 259 259 F F F
163102024-02-20T22:47:59.062Z 26 520-539 151 151 151 259 259 259 F F F
163112024-02-20T22:47:59.062Z 27 540-559 144 144 144 248 248 248 F F F
163122024-02-20T22:47:59.062Z 28 560-579 150 150 150 257 257 257 F F F
163132024-02-20T22:47:59.062Z 29 580-599 145 145 145 249 249 249 F F F
163142024-02-20T22:47:59.062ZMax gen: 151, Max flush: 259
163152024-02-20T22:47:59.062Z
163162024-02-20T22:47:59.062ZBegin loop 76
163172024-02-20T22:47:59.062ZDownstairs to restart: 2
163182024-02-20T22:47:59.080Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.590821199Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746}
163192024-02-20T22:47:59.080Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591178197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746}
163202024-02-20T22:47:59.080ZCrucible runtime is spawned
163212024-02-20T22:47:59.087Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591227935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746}
163222024-02-20T22:47:59.088Z{"msg":"Crucible 24e5ee0d-bfa3-4717-886a-79380b13f9dc has session id: 1bcb2642-b5ea-4cd1-9896-f30ef573d41d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591266984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163232024-02-20T22:47:59.088Z{"msg":"Upstairs opts: Upstairs UUID: 24e5ee0d-bfa3-4717-886a-79380b13f9dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591318752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163242024-02-20T22:47:59.088Z{"msg":"Crucible stats registered with UUID: 24e5ee0d-bfa3-4717-886a-79380b13f9dc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59135456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163252024-02-20T22:47:59.088Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59137504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746}
163262024-02-20T22:47:59.088Z{"msg":"24e5ee0d-bfa3-4717-886a-79380b13f9dc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591409858Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163272024-02-20T22:47:59.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591444197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163282024-02-20T22:47:59.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591481866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163292024-02-20T22:47:59.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591517695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163302024-02-20T22:47:59.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591556843Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163312024-02-20T22:47:59.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591583222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163322024-02-20T22:47:59.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591610141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163332024-02-20T22:47:59.088Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59163747Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163342024-02-20T22:47:59.088Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591670129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163352024-02-20T22:47:59.088Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591697768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163362024-02-20T22:47:59.088Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591725707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163372024-02-20T22:47:59.088Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591845333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163382024-02-20T22:47:59.088Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.591890481Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163392024-02-20T22:47:59.088Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.592542958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163402024-02-20T22:47:59.089Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.592583127Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163412024-02-20T22:47:59.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.592609906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163422024-02-20T22:47:59.089Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.592680293Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163432024-02-20T22:47:59.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.592728441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163442024-02-20T22:47:59.089Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59275701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163452024-02-20T22:47:59.089Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593243543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163462024-02-20T22:47:59.089Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593293391Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163472024-02-20T22:47:59.089Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.5933287Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163482024-02-20T22:47:59.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593787773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163492024-02-20T22:47:59.089Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593833282Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163502024-02-20T22:47:59.089Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593861141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163512024-02-20T22:47:59.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59388021Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163522024-02-20T22:47:59.089Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593895589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163532024-02-20T22:47:59.089Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593918129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163542024-02-20T22:47:59.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593933138Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163552024-02-20T22:47:59.089Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593954938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163562024-02-20T22:47:59.089Z{"msg":"[0]R flush_numbers[0..12]: [246, 259, 253, 253, 252, 259, 259, 246, 256, 252, 257, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593970337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163572024-02-20T22:47:59.089Z{"msg":"[0]R generation[0..12]: [141, 151, 148, 148, 147, 151, 151, 141, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.593991656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163582024-02-20T22:47:59.090Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594006746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163592024-02-20T22:47:59.090Z{"msg":"[1]R flush_numbers[0..12]: [246, 259, 253, 253, 252, 259, 259, 246, 256, 252, 257, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594028465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163602024-02-20T22:47:59.090Z{"msg":"[1]R generation[0..12]: [141, 151, 148, 148, 147, 151, 151, 141, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594042874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163612024-02-20T22:47:59.090Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
163622024-02-20T22:47:59.090Z,"time":"2024-02-20T22:47:58.594070253Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163632024-02-20T22:47:59.090Z{"msg":"[2]R flush_numbers[0..12]: [246, 259, 253, 253, 252, 259, 259, 246, 256, 252, 257, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594089513Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163642024-02-20T22:47:59.090Z{"msg":"[2]R generation[0..12]: [141, 151, 148, 148, 147, 151, 151, 141, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594113762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163652024-02-20T22:47:59.090Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594129261Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163662024-02-20T22:47:59.090Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.5941505Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163672024-02-20T22:47:59.090Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59416544Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163682024-02-20T22:47:59.090Z{"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594186689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163692024-02-20T22:47:59.090Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594222168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163702024-02-20T22:47:59.090Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594251027Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163712024-02-20T22:47:59.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594268196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163722024-02-20T22:47:59.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594303025Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163732024-02-20T22:47:59.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594337444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163742024-02-20T22:47:59.090Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594353833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163752024-02-20T22:47:59.090Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594380542Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163762024-02-20T22:47:59.091Z{"msg":"24e5ee0d-bfa3-4717-886a-79380b13f9dc is now active with session: 1bcb2642-b5ea-4cd1-9896-f30ef573d41d","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594417231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163772024-02-20T22:47:59.091Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.59444575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746,"session_id":"1bcb2642-b5ea-4cd1-9896-f30ef573d41d"}
163782024-02-20T22:47:59.091Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:47:58.594463849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746}
163792024-02-20T22:47:59.091ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163802024-02-20T22:47:59.091ZLoading write count information from file "/tmp/test_repair_verify.data"
163812024-02-20T22:47:59.091ZRun Repair workload
163822024-02-20T22:47:59.096Z01/30 Write block 464 len 32768 data: 16 15 11 13 16 19 15 12
163832024-02-20T22:47:59.096Z02/30 Write block 447 len 28672 data: 12 10 9 9 10 12 10
163842024-02-20T22:47:59.096Z03/30 Write block 365 len 20480 data: 18 17 12 12 14
163852024-02-20T22:47:59.096Z04/30 Write block 85 len 16384 data: 5 5 6 8
163862024-02-20T22:47:59.097Z05/30 Read block 482 len 28672
163872024-02-20T22:47:59.097Z06/30 Write block 502 len 32768 data: 8 7 10 12 11 12 11 11
163882024-02-20T22:47:59.097Z07/30 Read block 39 len 32768
163892024-02-20T22:47:59.097Z08/30 Write block 382 len 32768 data: 12 11 12 10 8 7 8 11
163902024-02-20T22:47:59.097Z09/30 Write block 20 len 32768 data: 13 9 12 10 9 14 12 17
163912024-02-20T22:47:59.097Z10/30 Write block 43 len 8192 data: 10 10
163922024-02-20T22:47:59.097Z11/30 Write block 400 len 4096 data: 13
163932024-02-20T22:47:59.097Z12/30 Read block 325 len 16384
163942024-02-20T22:47:59.097Z13/30 Read block 107 len 8192
163952024-02-20T22:47:59.097Z14/30 Write block 259 len 12288 data: 11 10 9
163962024-02-20T22:47:59.101Z15/30 Write block 304 len 36864 data: 17 17 13 13 12 13 12 12 11
163972024-02-20T22:47:59.101Z16/30 Read block 419 len 16384
163982024-02-20T22:47:59.101Z17/30 Write block 378 len 12288 data: 12 11 11
163992024-02-20T22:47:59.101Z18/30 Read block 585 len 28672
164002024-02-20T22:47:59.101Z19/30 Read block 447 len 28672
164012024-02-20T22:47:59.101Z20/30 Read block 249 len 40960
164022024-02-20T22:47:59.105Z21/30 Flush
164032024-02-20T22:47:59.109Z22/30 Write block 399 len 12288 data: 12 14 11
164042024-02-20T22:47:59.109Z23/30 Read block 175 len 28672
164052024-02-20T22:47:59.109Z24/30 Read block 588 len 40960
164062024-02-20T22:47:59.109Z25/30 Read block 456 len 12288
164072024-02-20T22:47:59.112Z26/30 Write block 285 len 4096 data: 11
164082024-02-20T22:47:59.112Z27/30 Read block 390 len 24576
164092024-02-20T22:47:59.112Z28/30 Read block 14 len 36864
164102024-02-20T22:47:59.112Z29/30 Write block 110 len 20480 data: 14 14 12 12 11
164112024-02-20T22:47:59.112Z30/30 Read block 286 len 24576
164122024-02-20T22:47:59.119Z----------------------------------------------------------------
164132024-02-20T22:47:59.119Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
164142024-02-20T22:47:59.119ZStates: Active Active Active
164152024-02-20T22:47:59.120ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164162024-02-20T22:47:59.120Z 1 Acked 1000 Write 8 Done Done Sent false
164172024-02-20T22:47:59.120Z 2 Acked 1001 Write 7 Done Done Sent false
164182024-02-20T22:47:59.120Z 3 Acked 1002 Write 5 Done Done Sent false
164192024-02-20T22:47:59.120Z 4 Acked 1003 Write 4 Done Done Sent false
164202024-02-20T22:47:59.120Z 5 Acked 1004 Read 7 Done Done Sent false
164212024-02-20T22:47:59.120Z 6 Acked 1005 Write 8 Done Done Sent false
164222024-02-20T22:47:59.120Z 7 Acked 1006 Read 8 Done Done Sent false
164232024-02-20T22:47:59.120Z 8 Acked 1007 Write 8 Done Done Sent false
164242024-02-20T22:47:59.120Z 9 Acked 1008 Write 8 Done Done Sent false
164252024-02-20T22:47:59.120Z 10 Acked 1009 Write 2 Done Done Sent false
164262024-02-20T22:47:59.120Z 11 Acked 1010 Write 1 Done Done Sent false
164272024-02-20T22:47:59.120Z 12 Acked 1011 Read 4 Done Done Sent false
164282024-02-20T22:47:59.120Z 13 Acked 1012 Read 2 Done Done Sent false
164292024-02-20T22:47:59.120Z 14 Acked 1013 Write 3 Done Done Sent false
164302024-02-20T22:47:59.120Z 15 Acked 1014 Write 9 Done Done Sent false
164312024-02-20T22:47:59.120Z 16 Acked 1015 Read 4 Done Done Sent false
164322024-02-20T22:47:59.120Z 17 Acked 1016 Write 3 Done Done Sent false
164332024-02-20T22:47:59.120Z 18 Acked 1017 Read 7 Done Done Sent false
164342024-02-20T22:47:59.120Z 19 Acked 1018 Read 7 Done Done Sent false
164352024-02-20T22:47:59.120Z 20 Acked 1019 Read 10 Done Done Sent false
164362024-02-20T22:47:59.120Z 21 Acked 1020 Flush 0 Done Done Sent false
164372024-02-20T22:47:59.121Z 22 Acked 1021 Write 3 Done Done Sent false
164382024-02-20T22:47:59.121Z 23 Acked 1022 Read 7 Done Done Sent false
164392024-02-20T22:47:59.121Z 24 Acked 1023 Read 10 Done Done Sent false
164402024-02-20T22:47:59.121Z 25 Acked 1024 Read 3 Done Done Sent false
164412024-02-20T22:47:59.121Z 26 Acked 1025 Write 1 Done Done Sent false
164422024-02-20T22:47:59.121Z 27 Acked 1026 Read 6 Done Done Sent false
164432024-02-20T22:47:59.121Z 28 Acked 1027 Read 9 Done Done Sent false
164442024-02-20T22:47:59.121Z 29 Acked 1028 Write 5 Done Done Sent false
164452024-02-20T22:47:59.121Z 30 Acked 1029 Read 6 Sent Done Sent false
164462024-02-20T22:47:59.121Z STATES DS:0 DS:1 DS:2 TOTAL
164472024-02-20T22:47:59.121Z New 0 0 0 0
164482024-02-20T22:47:59.121Z Sent 1 0 30 31
164492024-02-20T22:47:59.121Z Done 29 30 0 59
164502024-02-20T22:47:59.121Z Skipped 0 0 0 0
164512024-02-20T22:47:59.121Z Error 0 0 0 0
164522024-02-20T22:47:59.121ZLast Flush: 1020 1020 0
164532024-02-20T22:47:59.121ZDownstairs last five completed:
164542024-02-20T22:47:59.121ZUpstairs last five completed: 30 29 28 27 26
164552024-02-20T22:47:59.122Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:47:58.624840112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1746}
164562024-02-20T22:47:59.122ZWrote out file "/tmp/test_repair_verify.data"
164572024-02-20T22:47:59.126Z
164582024-02-20T22:47:59.126ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164592024-02-20T22:47:59.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164602024-02-20T22:47:59.139Z 0 000-019 141 141 141 246 246 246 F F F
164612024-02-20T22:47:59.139Z 1 020-039 152 152 151 260 260 259 F F F <---
164622024-02-20T22:47:59.139Z 2 040-059 152 152 148 260 260 253 F F F <---
164632024-02-20T22:47:59.140Z 3 060-079 148 148 148 253 253 253 F F F
164642024-02-20T22:47:59.140Z 4 080-099 152 152 147 260 260 252 F F F <---
164652024-02-20T22:47:59.140Z 5 100-119 151 151 151 259 259 259 T T F <---
164662024-02-20T22:47:59.140Z 6 120-139 151 151 151 259 259 259 F F F
164672024-02-20T22:47:59.140Z 7 140-159 141 141 141 246 246 246 F F F
164682024-02-20T22:47:59.141Z 8 160-179 149 149 149 256 256 256 F F F
164692024-02-20T22:47:59.154Z 9 180-199 147 147 147 252 252 252 F F F
164702024-02-20T22:47:59.154Z 10 200-219 150 150 150 257 257 257 F F F
164712024-02-20T22:47:59.154Z 11 220-239 150 150 150 258 258 258 F F F
164722024-02-20T22:47:59.154Z 12 240-259 152 152 144 260 260 248 F F F <---
164732024-02-20T22:47:59.154Z 13 260-279 152 152 150 260 260 258 F F F <---
164742024-02-20T22:47:59.154Z 14 280-299 148 148 148 253 253 253 T T F <---
164752024-02-20T22:47:59.154Z 15 300-319 152 152 148 260 260 254 F F F <---
164762024-02-20T22:47:59.154Z 16 320-339 150 150 150 257 257 257 F F F
164772024-02-20T22:47:59.155Z 17 340-359 151 151 151 259 259 259 F F F
164782024-02-20T22:47:59.155Z 18 360-379 152 152 151 260 260 259 F F F <---
164792024-02-20T22:47:59.155Z 19 380-399 152 152 148 260 260 255 T T F <---
164802024-02-20T22:47:59.155Z 20 400-419 152 152 150 260 260 257 T T F <---
164812024-02-20T22:47:59.155Z 21 420-439 148 148 148 255 255 255 F F F
164822024-02-20T22:47:59.155Z 22 440-459 152 152 150 260 260 257 F F F <---
164832024-02-20T22:47:59.155Z 23 460-479 152 152 141 260 260 246 F F F <---
164842024-02-20T22:47:59.155Z 24 480-499 149 149 149 256 256 256 F F F
164852024-02-20T22:47:59.155Z 25 500-519 152 152 151 260 260 259 F F F <---
164862024-02-20T22:47:59.155Z 26 520-539 151 151 151 259 259 259 F F F
164872024-02-20T22:47:59.155Z 27 540-559 144 144 144 248 248 248 F F F
164882024-02-20T22:47:59.155Z 28 560-579 150 150 150 257 257 257 F F F
164892024-02-20T22:47:59.155Z 29 580-599 145 145 145 249 249 249 F F F
164902024-02-20T22:47:59.155ZMax gen: 152, Max flush: 260
164912024-02-20T22:47:59.155ZError: Difference in extent metadata found!
164922024-02-20T22:47:59.155ZA Difference in extent metadata is expected here
164932024-02-20T22:47:59.156ZOn loop 76
164942024-02-20T22:47:59.156Z
164952024-02-20T22:47:59.156ZVerifying data now
164962024-02-20T22:48:03.826ZNo change after verify
164972024-02-20T22:48:03.826ZLoop: 76 Downstairs dump after verify (and repair):
164982024-02-20T22:48:03.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164992024-02-20T22:48:03.843Z 0 000-019 141 141 141 246 246 246 F F F
165002024-02-20T22:48:03.843Z 1 020-039 152 152 152 260 260 260 F F F
165012024-02-20T22:48:03.843Z 2 040-059 152 152 152 260 260 260 F F F
165022024-02-20T22:48:03.843Z 3 060-079 148 148 148 253 253 253 F F F
165032024-02-20T22:48:03.843Z 4 080-099 152 152 152 260 260 260 F F F
165042024-02-20T22:48:03.843Z 5 100-119 153 153 153 261 261 261 F F F
165052024-02-20T22:48:03.848Z 6 120-139 151 151 151 259 259 259 F F F
165062024-02-20T22:48:03.848Z 7 140-159 141 141 141 246 246 246 F F F
165072024-02-20T22:48:03.848Z 8 160-179 149 149 149 256 256 256 F F F
165082024-02-20T22:48:03.848Z 9 180-199 147 147 147 252 252 252 F F F
165092024-02-20T22:48:03.848Z 10 200-219 150 150 150 257 257 257 F F F
165102024-02-20T22:48:03.848Z 11 220-239 150 150 150 258 258 258 F F F
165112024-02-20T22:48:03.848Z 12 240-259 152 152 152 260 260 260 F F F
165122024-02-20T22:48:03.848Z 13 260-279 152 152 152 260 260 260 F F F
165132024-02-20T22:48:03.848Z 14 280-299 153 153 153 261 261 261 F F F
165142024-02-20T22:48:03.848Z 15 300-319 152 152 152 260 260 260 F F F
165152024-02-20T22:48:03.848Z 16 320-339 150 150 150 257 257 257 F F F
165162024-02-20T22:48:03.849Z 17 340-359 151 151 151 259 259 259 F F F
165172024-02-20T22:48:03.849Z 18 360-379 152 152 152 260 260 260 F F F
165182024-02-20T22:48:03.849Z 19 380-399 153 153 153 261 261 261 F F F
165192024-02-20T22:48:03.849Z 20 400-419 153 153 153 261 261 261 F F F
165202024-02-20T22:48:03.849Z 21 420-439 148 148 148 255 255 255 F F F
165212024-02-20T22:48:03.849Z 22 440-459 152 152 152 260 260 260 F F F
165222024-02-20T22:48:03.849Z 23 460-479 152 152 152 260 260 260 F F F
165232024-02-20T22:48:03.849Z 24 480-499 149 149 149 256 256 256 F F F
165242024-02-20T22:48:03.849Z 25 500-519 152 152 152 260 260 260 F F F
165252024-02-20T22:48:03.849Z 26 520-539 151 151 151 259 259 259 F F F
165262024-02-20T22:48:03.849Z 27 540-559 144 144 144 248 248 248 F F F
165272024-02-20T22:48:03.849Z 28 560-579 150 150 150 257 257 257 F F F
165282024-02-20T22:48:03.849Z 29 580-599 145 145 145 249 249 249 F F F
165292024-02-20T22:48:03.849ZMax gen: 153, Max flush: 261
165302024-02-20T22:48:03.849Z
165312024-02-20T22:48:03.849ZBegin loop 77
165322024-02-20T22:48:03.849ZDownstairs to restart: 0
165332024-02-20T22:48:03.862Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.371699126Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754}
165342024-02-20T22:48:03.862Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.371939118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754}
165352024-02-20T22:48:03.862Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.371966057Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754}
165362024-02-20T22:48:03.871Z{"msg":"Crucible bbeb7acc-7c81-417d-8e49-d965f7498ee7 has session id: 906e0927-56c0-4129-9be7-433e194a1367","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.371982456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165372024-02-20T22:48:03.871Z{"msg":"Upstairs opts: Upstairs UUID: bbeb7acc-7c81-417d-8e49-d965f7498ee7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372029074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165382024-02-20T22:48:03.871Z{"msg":"Crucible stats registered with UUID: bbeb7acc-7c81-417d-8e49-d965f7498ee7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372067063Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal"Crucible runtime is spawned
165392024-02-20T22:48:03.871Z,"pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165402024-02-20T22:48:03.871Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372198658Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754}
165412024-02-20T22:48:03.871Z{"msg":"bbeb7acc-7c81-417d-8e49-d965f7498ee7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372242927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165422024-02-20T22:48:03.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372301625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165432024-02-20T22:48:03.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372346793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165442024-02-20T22:48:03.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372365362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165452024-02-20T22:48:03.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372393531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165462024-02-20T22:48:03.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372468869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165472024-02-20T22:48:03.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372522327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165482024-02-20T22:48:03.871Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372561965Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165492024-02-20T22:48:03.872Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372623593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165502024-02-20T22:48:03.872Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372666092Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165512024-02-20T22:48:03.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372691551Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165522024-02-20T22:48:03.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372727369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165532024-02-20T22:48:03.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.372753208Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165542024-02-20T22:48:03.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373392785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165552024-02-20T22:48:03.872Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373429294Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165562024-02-20T22:48:03.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373445964Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165572024-02-20T22:48:03.872Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373461083Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165582024-02-20T22:48:03.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373500632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165592024-02-20T22:48:03.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373521011Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165602024-02-20T22:48:03.872Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.373974515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165612024-02-20T22:48:03.872Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374016673Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165622024-02-20T22:48:03.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374209646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165632024-02-20T22:48:03.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374430928Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165642024-02-20T22:48:03.872Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374467537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165652024-02-20T22:48:03.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374493426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165662024-02-20T22:48:03.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374592863Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165672024-02-20T22:48:03.872Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374629601Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165682024-02-20T22:48:03.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.37466434Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165692024-02-20T22:48:03.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374706239Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165702024-02-20T22:48:03.872Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374729598Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165712024-02-20T22:48:03.872Z{"msg":"[0]R flush_numbers[0..12]: [246, 260, 260, 253, 260, 261, 259, 246, 256, 252, 257, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374749207Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165722024-02-20T22:48:03.872Z{"msg":"[0]R generation[0..12]: [141, 152, 152, 148, 152, 153, 151, 141, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374763867Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165732024-02-20T22:48:03.872Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374787006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165742024-02-20T22:48:03.873Z{"msg":"[1]R flush_numbers[0..12]: [246, 260, 260, 253, 260, 261, 259, 246, 256, 252, 257, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374802345Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165752024-02-20T22:48:03.873Z{"msg":"[1]R generation[0..12]: [141, 152, 152, 148, 152, 153, 151, 141, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374823125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165762024-02-20T22:48:03.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374837474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165772024-02-20T22:48:03.873Z{"msg":"[2]R flush_numbers[0..12]: [246, 260, 260, 253, 260, 261, 259, 246, 256, 252, 257, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374857863Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165782024-02-20T22:48:03.873ZWait for a query_work_queue command to finish before sending IO
165792024-02-20T22:48:03.873Z{"msg":"[2]R generation[0..12]: [141, 152, 152, 148, 152, 153, 151, 141, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374879532Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165802024-02-20T22:48:03.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374900692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165812024-02-20T22:48:03.873Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374914351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165822024-02-20T22:48:03.873Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.37493586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165832024-02-20T22:48:03.873Z{"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.37494901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165842024-02-20T22:48:03.873Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374963089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165852024-02-20T22:48:03.873Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374976729Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165862024-02-20T22:48:03.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.374995508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165872024-02-20T22:48:03.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375009118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165882024-02-20T22:48:03.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375029317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165892024-02-20T22:48:03.873Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375042826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165902024-02-20T22:48:03.873Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375055206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165912024-02-20T22:48:03.873Z{"msg":"bbeb7acc-7c81-417d-8e49-d965f7498ee7 is now active with session: 906e0927-56c0-4129-9be7-433e194a1367","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375068646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165922024-02-20T22:48:03.873Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375087185Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754,"session_id":"906e0927-56c0-4129-9be7-433e194a1367"}
165932024-02-20T22:48:03.873Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:03.375100175Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754}
165942024-02-20T22:48:03.873ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165952024-02-20T22:48:03.873ZLoading write count information from file "/tmp/test_repair_verify.data"
165962024-02-20T22:48:03.873ZRun Repair workload
165972024-02-20T22:48:03.873Z01/30 Flush
165982024-02-20T22:48:03.874Z02/30 Write block 29 len 36864 data: 22 21 21 22 23 21 22 19 17
165992024-02-20T22:48:03.879Z03/30 Flush
166002024-02-20T22:48:03.879Z04/30 Flush
166012024-02-20T22:48:03.879Z05/30 Write block 30 len 24576 data: 22 22 23 24 22 23
166022024-02-20T22:48:03.879Z06/30 Read block 37 len 28672
166032024-02-20T22:48:03.879Z07/30 Read block 294 len 28672
166042024-02-20T22:48:03.879Z08/30 Write block 187 len 40960 data: 12 10 10 10 10 10 9 9 12 10
166052024-02-20T22:48:03.879Z09/30 Write block 33 len 12288 data: 25 23 24
166062024-02-20T22:48:03.879Z10/30 Read block 459 len 8192
166072024-02-20T22:48:03.879Z11/30 Write block 389 len 24576 data: 12 12 11 13 12 10
166082024-02-20T22:48:03.880Z12/30 Flush
166092024-02-20T22:48:03.884Z13/30 Write block 416 len 40960 data: 14 12 13 16 16 12 10 10 8 10
166102024-02-20T22:48:03.884Z14/30 Read block 65 len 32768
166112024-02-20T22:48:03.884Z15/30 Write block 474 len 32768 data: 11 11 13 17 19 15 18 17
166122024-02-20T22:48:03.884Z16/30 Write block 122 len 12288 data: 14 12 12
166132024-02-20T22:48:03.884Z17/30 Read block 177 len 20480
166142024-02-20T22:48:03.888Z18/30 Write block 363 len 16384 data: 20 18 19 18
166152024-02-20T22:48:03.888Z19/30 Read block 17 len 12288
166162024-02-20T22:48:03.888Z20/30 Write block 321 len 32768 data: 14 17 13 11 16 17 15 14
166172024-02-20T22:48:03.888Z21/30 Write block 547 len 12288 data: 11 10 10
166182024-02-20T22:48:03.888Z22/30 Read block 257 len 12288
166192024-02-20T22:48:03.894Z23/30 Write block 174 len 36864 data: 10 13 12 13 13 16 15 18 16
166202024-02-20T22:48:03.894Z24/30 Read block 143 len 40960
166212024-02-20T22:48:03.894Z25/30 Write block 241 len 16384 data: 7 6 6 6
166222024-02-20T22:48:03.894Z26/30 Write block 195 len 36864 data: 13 11 12 11 11 7 6 6 8
166232024-02-20T22:48:03.894Z27/30 Read block 413 len 8192
166242024-02-20T22:48:03.899Z28/30 Write block 3 len 32768 data: 10 12 12 7 6 6 7 10
166252024-02-20T22:48:03.899Z29/30 Write block 456 len 32768 data: 17 17 19 18 16 18 18 19
166262024-02-20T22:48:03.899Z30/30 Read block 491 len 28672
166272024-02-20T22:48:03.899Z----------------------------------------------------------------
166282024-02-20T22:48:03.899Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166292024-02-20T22:48:03.899ZStates: Active Active Active
166302024-02-20T22:48:03.899ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166312024-02-20T22:48:03.899Z 1 Acked 1000 Flush 0 Sent Done Done false
166322024-02-20T22:48:03.900Z 2 Acked 1001 Write 9 Sent Done Done false
166332024-02-20T22:48:03.900Z 3 Acked 1002 Flush 0 Sent Done Done false
166342024-02-20T22:48:03.900Z 4 Acked 1003 Flush 0 Sent Done Done false
166352024-02-20T22:48:03.900Z 5 Acked 1004 Write 6 Sent Done Done false
166362024-02-20T22:48:03.900Z 6 Acked 1005 Read 7 Sent Done Done false
166372024-02-20T22:48:03.900Z 7 Acked 1006 Read 7 Sent Done Done false
166382024-02-20T22:48:03.900Z 8 Acked 1007 Write 10 Sent Done Done false
166392024-02-20T22:48:03.900Z 9 Acked 1008 Write 3 Sent Done Done false
166402024-02-20T22:48:03.900Z 10 Acked 1009 Read 2 Sent Done Done false
166412024-02-20T22:48:03.900Z 11 Acked 1010 Write 6 Sent Done Done false
166422024-02-20T22:48:03.900Z 12 Acked 1011 Flush 0 Sent Done Done false
166432024-02-20T22:48:03.900Z 13 Acked 1012 Write 10 Sent Done Done false
166442024-02-20T22:48:03.900Z 14 Acked 1013 Read 8 Sent Done Done false
166452024-02-20T22:48:03.900Z 15 Acked 1014 Write 8 Sent Done Done false
166462024-02-20T22:48:03.900Z 16 Acked 1015 Write 3 Sent Done Done false
166472024-02-20T22:48:03.900Z 17 Acked 1016 Read 5 Sent Done Done false
166482024-02-20T22:48:03.900Z 18 Acked 1017 Write 4 Sent Done Done false
166492024-02-20T22:48:03.900Z 19 Acked 1018 Read 3 Sent Done Done false
166502024-02-20T22:48:03.900Z 20 Acked 1019 Write 8 Sent Done Done false
166512024-02-20T22:48:03.900Z 21 Acked 1020 Write 3 Sent Done Done false
166522024-02-20T22:48:03.900Z 22 Acked 1021 Read 3 Sent Done Done false
166532024-02-20T22:48:03.900Z 23 Acked 1022 Write 9 Sent Done Done false
166542024-02-20T22:48:03.900Z 24 Acked 1023 Read 10 Sent Done Done false
166552024-02-20T22:48:03.900Z 25 Acked 1024 Write 4 Sent Done Done false
166562024-02-20T22:48:03.900Z 26 Acked 1025 Write 9 Sent Done Done false
166572024-02-20T22:48:03.901Z 27 Acked 1026 Read 2 Sent Done Done false
166582024-02-20T22:48:03.901Z 28 Acked 1027 Write 8 Sent Done Done false
166592024-02-20T22:48:03.901Z 29 Acked 1028 Write 8 Sent Done Done false
166602024-02-20T22:48:03.901Z 30 Acked 1029 Read 7 Sent Done Sent false
166612024-02-20T22:48:03.901Z STATES DS:0 DS:1 DS:2 TOTAL
166622024-02-20T22:48:03.901Z New 0 0 0 0
166632024-02-20T22:48:03.901Z Sent 30 0 1 31
166642024-02-20T22:48:03.901Z Done 0 30 29 59
166652024-02-20T22:48:03.901Z Skipped 0 0 0 0
166662024-02-20T22:48:03.901Z Error 0 0 0 0
166672024-02-20T22:48:03.901ZLast Flush: 0 1011 1011
166682024-02-20T22:48:03.901ZDownstairs last five completed:
166692024-02-20T22:48:03.901ZUpstairs last five completed: 30 29 28 27 26
166702024-02-20T22:48:03.901Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:03.407242654Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1754}
166712024-02-20T22:48:03.901ZWrote out file "/tmp/test_repair_verify.data"
166722024-02-20T22:48:03.907Z
166732024-02-20T22:48:03.907ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166742024-02-20T22:48:03.923ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166752024-02-20T22:48:03.923Z 0 000-019 141 141 141 246 246 246 F T T <---
166762024-02-20T22:48:03.923Z 1 020-039 152 154 154 260 265 265 F F F <---
166772024-02-20T22:48:03.923Z 2 040-059 152 152 152 260 260 260 F F F
166782024-02-20T22:48:03.923Z 3 060-079 148 148 148 253 253 253 F F F
166792024-02-20T22:48:03.923Z 4 080-099 152 152 152 260 260 260 F F F
166802024-02-20T22:48:03.923Z 5 100-119 153 153 153 261 261 261 F F F
166812024-02-20T22:48:03.923Z 6 120-139 151 151 151 259 259 259 F T T <---
166822024-02-20T22:48:03.923Z 7 140-159 141 141 141 246 246 246 F F F
166832024-02-20T22:48:03.923Z 8 160-179 149 149 149 256 256 256 F T T <---
166842024-02-20T22:48:03.923Z 9 180-199 147 154 154 252 265 265 F T T <---
166852024-02-20T22:48:03.923Z 10 200-219 150 150 150 257 257 257 F T T <---
166862024-02-20T22:48:03.923Z 11 220-239 150 150 150 258 258 258 F F F
166872024-02-20T22:48:03.927Z 12 240-259 152 152 152 260 260 260 F T T <---
166882024-02-20T22:48:03.928Z 13 260-279 152 152 152 260 260 260 F F F
166892024-02-20T22:48:03.928Z 14 280-299 153 153 153 261 261 261 F F F
166902024-02-20T22:48:03.928Z 15 300-319 152 152 152 260 260 260 F F F
166912024-02-20T22:48:03.928Z 16 320-339 150 150 150 257 257 257 F T T <---
166922024-02-20T22:48:03.928Z 17 340-359 151 151 151 259 259 259 F F F
166932024-02-20T22:48:03.928Z 18 360-379 152 152 152 260 260 260 F T T <---
166942024-02-20T22:48:03.928Z 19 380-399 153 154 154 261 265 265 F F F <---
166952024-02-20T22:48:03.928Z 20 400-419 153 153 153 261 261 261 F T T <---
166962024-02-20T22:48:03.928Z 21 420-439 148 148 148 255 255 255 F T T <---
166972024-02-20T22:48:03.928Z 22 440-459 152 152 152 260 260 260 F T T <---
166982024-02-20T22:48:03.928Z 23 460-479 152 152 152 260 260 260 F T T <---
166992024-02-20T22:48:03.928Z 24 480-499 149 149 149 256 256 256 F T T <---
167002024-02-20T22:48:03.928Z 25 500-519 152 152 152 260 260 260 F F F
167012024-02-20T22:48:03.928Z 26 520-539 151 151 151 259 259 259 F F F
167022024-02-20T22:48:03.928Z 27 540-559 144 144 144 248 248 248 F T T <---
167032024-02-20T22:48:03.928Z 28 560-579 150 150 150 257 257 257 F F F
167042024-02-20T22:48:03.928Z 29 580-599 145 145 145 249 249 249 F F F
167052024-02-20T22:48:03.928ZMax gen: 154, Max flush: 265
167062024-02-20T22:48:03.928ZError: Difference in extent metadata found!
167072024-02-20T22:48:03.928ZA Difference in extent metadata is expected here
167082024-02-20T22:48:03.928ZOn loop 77
167092024-02-20T22:48:03.928Z
167102024-02-20T22:48:03.934ZVerifying data now
167112024-02-20T22:48:08.713ZNo change after verify
167122024-02-20T22:48:08.713ZLoop: 77 Downstairs dump after verify (and repair):
167132024-02-20T22:48:08.727ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167142024-02-20T22:48:08.727Z 0 000-019 155 155 155 266 266 266 F F F
167152024-02-20T22:48:08.728Z 1 020-039 154 154 154 265 265 265 F F F
167162024-02-20T22:48:08.728Z 2 040-059 152 152 152 260 260 260 F F F
167172024-02-20T22:48:08.728Z 3 060-079 148 148 148 253 253 253 F F F
167182024-02-20T22:48:08.728Z 4 080-099 152 152 152 260 260 260 F F F
167192024-02-20T22:48:08.728Z 5 100-119 153 153 153 261 261 261 F F F
167202024-02-20T22:48:08.728Z 6 120-139 155 155 155 266 266 266 F F F
167212024-02-20T22:48:08.728Z 7 140-159 141 141 141 246 246 246 F F F
167222024-02-20T22:48:08.728Z 8 160-179 155 155 155 266 266 266 F F F
167232024-02-20T22:48:08.728Z 9 180-199 155 155 155 266 266 266 F F F
167242024-02-20T22:48:08.728Z 10 200-219 155 155 155 266 266 266 F F F
167252024-02-20T22:48:08.728Z 11 220-239 150 150 150 258 258 258 F F F
167262024-02-20T22:48:08.733Z 12 240-259 155 155 155 266 266 266 F F F
167272024-02-20T22:48:08.733Z 13 260-279 152 152 152 260 260 260 F F F
167282024-02-20T22:48:08.733Z 14 280-299 153 153 153 261 261 261 F F F
167292024-02-20T22:48:08.733Z 15 300-319 152 152 152 260 260 260 F F F
167302024-02-20T22:48:08.733Z 16 320-339 155 155 155 266 266 266 F F F
167312024-02-20T22:48:08.733Z 17 340-359 151 151 151 259 259 259 F F F
167322024-02-20T22:48:08.733Z 18 360-379 155 155 155 266 266 266 F F F
167332024-02-20T22:48:08.733Z 19 380-399 154 154 154 265 265 265 F F F
167342024-02-20T22:48:08.733Z 20 400-419 155 155 155 266 266 266 F F F
167352024-02-20T22:48:08.733Z 21 420-439 155 155 155 266 266 266 F F F
167362024-02-20T22:48:08.733Z 22 440-459 155 155 155 266 266 266 F F F
167372024-02-20T22:48:08.733Z 23 460-479 155 155 155 266 266 266 F F F
167382024-02-20T22:48:08.733Z 24 480-499 155 155 155 266 266 266 F F F
167392024-02-20T22:48:08.733Z 25 500-519 152 152 152 260 260 260 F F F
167402024-02-20T22:48:08.733Z 26 520-539 151 151 151 259 259 259 F F F
167412024-02-20T22:48:08.734Z 27 540-559 155 155 155 266 266 266 F F F
167422024-02-20T22:48:08.734Z 28 560-579 150 150 150 257 257 257 F F F
167432024-02-20T22:48:08.734Z 29 580-599 145 145 145 249 249 249 F F F
167442024-02-20T22:48:08.734ZMax gen: 155, Max flush: 266
167452024-02-20T22:48:08.734Z
167462024-02-20T22:48:08.734ZBegin loop 78
167472024-02-20T22:48:08.734ZDownstairs to restart: 0
167482024-02-20T22:48:08.749Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259346572Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762}
167492024-02-20T22:48:08.758Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259659191Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762}
167502024-02-20T22:48:08.758Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.25969215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762}
167512024-02-20T22:48:08.758Z{"msg":"Crucible runtime is spawned
167522024-02-20T22:48:08.759ZCrucible 7d589e1a-9d62-49f9-b2dc-332710d8c582 has session id: e8a2d8b9-154e-48f9-93dc-287ffe473f87","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259726919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167532024-02-20T22:48:08.759Z{"msg":"Upstairs opts: Upstairs UUID: 7d589e1a-9d62-49f9-b2dc-332710d8c582, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259768297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167542024-02-20T22:48:08.759Z{"msg":"Crucible stats registered with UUID: 7d589e1a-9d62-49f9-b2dc-332710d8c582","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259810306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167552024-02-20T22:48:08.759Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259839265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762}
167562024-02-20T22:48:08.759Z{"msg":"7d589e1a-9d62-49f9-b2dc-332710d8c582 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.259986329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167572024-02-20T22:48:08.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260024998Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167582024-02-20T22:48:08.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260055757Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167592024-02-20T22:48:08.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260085946Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167602024-02-20T22:48:08.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260115375Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167612024-02-20T22:48:08.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260136974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167622024-02-20T22:48:08.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260162143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167632024-02-20T22:48:08.759Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260193652Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167642024-02-20T22:48:08.759Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.26023984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167652024-02-20T22:48:08.759Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.26025866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167662024-02-20T22:48:08.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260293388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167672024-02-20T22:48:08.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260320437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167682024-02-20T22:48:08.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.260402974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167692024-02-20T22:48:08.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261113549Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167702024-02-20T22:48:08.759Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261152348Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167712024-02-20T22:48:08.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261170597Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167722024-02-20T22:48:08.759Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261194336Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167732024-02-20T22:48:08.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261230725Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167742024-02-20T22:48:08.759Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261262794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167752024-02-20T22:48:08.760Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261724067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167762024-02-20T22:48:08.760Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.261764156Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167772024-02-20T22:48:08.760Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262037926Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167782024-02-20T22:48:08.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262303706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167792024-02-20T22:48:08.760Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262342975Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167802024-02-20T22:48:08.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262369004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167812024-02-20T22:48:08.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262409123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167822024-02-20T22:48:08.760Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262435462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167832024-02-20T22:48:08.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262459401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167842024-02-20T22:48:08.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.26248856Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167852024-02-20T22:48:08.760Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262513519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167862024-02-20T22:48:08.760Z{"msg":"[0]R flush_numbers[0..12]: [266, 265, 260, 253, 260, 261, 266, 246, 266, 266, 266, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262545228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167872024-02-20T22:48:08.760Z{"msg":"[0]R generation[0..12]: [155, 154, 152, 148, 152, 153, 155, 141, 155, 155, 155, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262578967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167882024-02-20T22:48:08.760Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262606096Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167892024-02-20T22:48:08.760Z{"msg":"[1]R flush_numbers[0..12]: [266, 265, 260, 253, 260, 261, 266, 246, 266, 266, 266, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262639385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167902024-02-20T22:48:08.760Z{"msg":"[1]R generation[0..12]: [155, 154, 152, 148, 152, 153, 155, 141, 155, 155, 155, 150]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
167912024-02-20T22:48:08.760Z,"time":"2024-02-20T22:48:08.262665333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167922024-02-20T22:48:08.760Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262701232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167932024-02-20T22:48:08.760Z{"msg":"[2]R flush_numbers[0..12]: [266, 265, 260, 253, 260, 261, 266, 246, 266, 266, 266, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262741071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167942024-02-20T22:48:08.760Z{"msg":"[2]R generation[0..12]: [155, 154, 152, 148, 152, 153, 155, 141, 155, 155, 155, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262775189Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167952024-02-20T22:48:08.760Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262796809Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167962024-02-20T22:48:08.760Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262828828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167972024-02-20T22:48:08.760Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262848187Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167982024-02-20T22:48:08.760Z{"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262878206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
167992024-02-20T22:48:08.760Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262893545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168002024-02-20T22:48:08.760Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262923924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168012024-02-20T22:48:08.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262939314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168022024-02-20T22:48:08.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262955403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168032024-02-20T22:48:08.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.262985252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168042024-02-20T22:48:08.761Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.263001072Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168052024-02-20T22:48:08.761Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.26303103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168062024-02-20T22:48:08.761Z{"msg":"7d589e1a-9d62-49f9-b2dc-332710d8c582 is now active with session: e8a2d8b9-154e-48f9-93dc-287ffe473f87","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.26304816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168072024-02-20T22:48:08.761Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.263077419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762,"session_id":"e8a2d8b9-154e-48f9-93dc-287ffe473f87"}
168082024-02-20T22:48:08.761Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:08.263091998Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762}
168092024-02-20T22:48:08.761ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168102024-02-20T22:48:08.761ZLoading write count information from file "/tmp/test_repair_verify.data"
168112024-02-20T22:48:08.761ZRun Repair workload
168122024-02-20T22:48:08.766Z01/30 Write block 255 len 12288 data: 9 8 9
168132024-02-20T22:48:08.766Z02/30 Read block 38 len 20480
168142024-02-20T22:48:08.766Z03/30 Read block 33 len 20480
168152024-02-20T22:48:08.766Z04/30 Read block 299 len 12288
168162024-02-20T22:48:08.766Z05/30 Read block 112 len 24576
168172024-02-20T22:48:08.766Z06/30 Flush
168182024-02-20T22:48:08.766Z07/30 Write block 31 len 12288 data: 23 24 26
168192024-02-20T22:48:08.766Z08/30 Write block 556 len 8192 data: 9 8
168202024-02-20T22:48:08.766Z09/30 Read block 465 len 12288
168212024-02-20T22:48:08.766Z10/30 Read block 214 len 28672
168222024-02-20T22:48:08.766Z11/30 Write block 121 len 20480 data: 13 15 13 13 9
168232024-02-20T22:48:08.766Z12/30 Read block 500 len 20480
168242024-02-20T22:48:08.766Z13/30 Write block 533 len 28672 data: 9 12 15 15 13 17 14
168252024-02-20T22:48:08.766Z14/30 Read block 5 len 36864
168262024-02-20T22:48:08.771Z15/30 Write block 307 len 8192 data: 14 13
168272024-02-20T22:48:08.771Z16/30 Write block 238 len 24576 data: 10 9 7 8 7 7
168282024-02-20T22:48:08.771Z17/30 Read block 7 len 40960
168292024-02-20T22:48:08.772Z18/30 Read block 275 len 12288
168302024-02-20T22:48:08.772Z19/30 Write block 425 len 8192 data: 11 9
168312024-02-20T22:48:08.772Z20/30 Read block 459 len 8192
168322024-02-20T22:48:08.776Z21/30 Read block 350 len 36864
168332024-02-20T22:48:08.776Z22/30 Write block 434 len 12288 data: 7 6 6
168342024-02-20T22:48:08.776Z23/30 Read block 597 len 4096
168352024-02-20T22:48:08.776Z24/30 Read block 222 len 4096
168362024-02-20T22:48:08.776Z25/30 Write block 424 len 40960 data: 9 12 10 8 9 9 8 7 5 6
168372024-02-20T22:48:08.776Z26/30 Flush
168382024-02-20T22:48:08.780Z27/30 Write block 361 len 24576 data: 15 18 21 19 20 19
168392024-02-20T22:48:08.780Z28/30 Write block 313 len 4096 data: 10
168402024-02-20T22:48:08.780Z29/30 Read block 573 len 40960
168412024-02-20T22:48:08.786Z30/30 Read block 385 len 20480
168422024-02-20T22:48:08.786Z----------------------------------------------------------------
168432024-02-20T22:48:08.786Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
168442024-02-20T22:48:08.786ZStates: Active Active Active
168452024-02-20T22:48:08.786ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168462024-02-20T22:48:08.786Z 1 Acked 1000 Write 3 Done Done Done false
168472024-02-20T22:48:08.786Z 2 Acked 1001 Read 5 Done Done Done false
168482024-02-20T22:48:08.786Z 3 Acked 1002 Read 5 Done Done Done false
168492024-02-20T22:48:08.786Z 4 Acked 1003 Read 3 Done Done Done false
168502024-02-20T22:48:08.786Z 5 Acked 1004 Read 6 Done Done Done false
168512024-02-20T22:48:08.786Z 6 Acked 1005 Flush 0 Sent Done Done false
168522024-02-20T22:48:08.786Z 7 Acked 1006 Write 3 Sent Done Done false
168532024-02-20T22:48:08.786Z 8 Acked 1007 Write 2 Sent Done Done false
168542024-02-20T22:48:08.786Z 9 Acked 1008 Read 3 Sent Done Done false
168552024-02-20T22:48:08.786Z 10 Acked 1009 Read 7 Sent Done Done false
168562024-02-20T22:48:08.786Z 11 Acked 1010 Write 5 Sent Done Done false
168572024-02-20T22:48:08.786Z 12 Acked 1011 Read 5 Sent Done Done false
168582024-02-20T22:48:08.786Z 13 Acked 1012 Write 7 Sent Done Done false
168592024-02-20T22:48:08.786Z 14 Acked 1013 Read 9 Sent Done Done false
168602024-02-20T22:48:08.786Z 15 Acked 1014 Write 2 Sent Done Done false
168612024-02-20T22:48:08.787Z 16 Acked 1015 Write 6 Sent Done Done false
168622024-02-20T22:48:08.787Z 17 Acked 1016 Read 10 Sent Done Done false
168632024-02-20T22:48:08.787Z 18 Acked 1017 Read 3 Sent Done Done false
168642024-02-20T22:48:08.787Z 19 Acked 1018 Write 2 Sent Done Done false
168652024-02-20T22:48:08.787Z 20 Acked 1019 Read 2 Sent Done Done false
168662024-02-20T22:48:08.787Z 21 Acked 1020 Read 9 Sent Done Done false
168672024-02-20T22:48:08.787Z 22 Acked 1021 Write 3 Sent Done Done false
168682024-02-20T22:48:08.787Z 23 Acked 1022 Read 1 Sent Done Done false
168692024-02-20T22:48:08.787Z 24 Acked 1023 Read 1 Sent Done Done false
168702024-02-20T22:48:08.787Z 25 Acked 1024 Write 10 Sent Done Done false
168712024-02-20T22:48:08.787Z 26 Acked 1025 Flush 0 Sent Done Done false
168722024-02-20T22:48:08.787Z 27 Acked 1026 Write 6 Sent Done Done false
168732024-02-20T22:48:08.787Z 28 Acked 1027 Write 1 Sent Done Done false
168742024-02-20T22:48:08.787Z 29 Acked 1028 Read 10 Sent Done Done false
168752024-02-20T22:48:08.787Z 30 Acked 1029 Read 5 Sent Sent Done false
168762024-02-20T22:48:08.787Z STATES DS:0 DS:1 DS:2 TOTAL
168772024-02-20T22:48:08.787Z New 0 0 0 0
168782024-02-20T22:48:08.787Z Sent 25 1 0 26
168792024-02-20T22:48:08.787Z Done 5 29 30 64
168802024-02-20T22:48:08.787Z Skipped 0 0 0 0
168812024-02-20T22:48:08.787Z Error 0 0 0 0
168822024-02-20T22:48:08.787ZLast Flush: 0 1025 1025
168832024-02-20T22:48:08.787ZDownstairs last five completed:
168842024-02-20T22:48:08.787ZUpstairs last five completed: 30 29 28 27 26
168852024-02-20T22:48:08.787Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:08.292559574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1762}
168862024-02-20T22:48:08.788ZWrote out file "/tmp/test_repair_verify.data"
168872024-02-20T22:48:08.792Z
168882024-02-20T22:48:08.792ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168892024-02-20T22:48:08.804ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168902024-02-20T22:48:08.804Z 0 000-019 155 155 155 266 266 266 F F F
168912024-02-20T22:48:08.805Z 1 020-039 154 156 156 265 268 268 F F F <---
168922024-02-20T22:48:08.805Z 2 040-059 152 152 152 260 260 260 F F F
168932024-02-20T22:48:08.805Z 3 060-079 148 148 148 253 253 253 F F F
168942024-02-20T22:48:08.805Z 4 080-099 152 152 152 260 260 260 F F F
168952024-02-20T22:48:08.805Z 5 100-119 153 153 153 261 261 261 F F F
168962024-02-20T22:48:08.805Z 6 120-139 155 156 156 266 268 268 F F F <---
168972024-02-20T22:48:08.805Z 7 140-159 141 141 141 246 246 246 F F F
168982024-02-20T22:48:08.805Z 8 160-179 155 155 155 266 266 266 F F F
168992024-02-20T22:48:08.809Z 9 180-199 155 155 155 266 266 266 F F F
169002024-02-20T22:48:08.809Z 10 200-219 155 155 155 266 266 266 F F F
169012024-02-20T22:48:08.809Z 11 220-239 150 156 156 258 268 268 F F F <---
169022024-02-20T22:48:08.809Z 12 240-259 155 156 156 266 268 268 T F F <---
169032024-02-20T22:48:08.809Z 13 260-279 152 152 152 260 260 260 F F F
169042024-02-20T22:48:08.809Z 14 280-299 153 153 153 261 261 261 F F F
169052024-02-20T22:48:08.810Z 15 300-319 152 156 156 260 268 268 F T T <---
169062024-02-20T22:48:08.810Z 16 320-339 155 155 155 266 266 266 F F F
169072024-02-20T22:48:08.810Z 17 340-359 151 151 151 259 259 259 F F F
169082024-02-20T22:48:08.810Z 18 360-379 155 155 155 266 266 266 F T T <---
169092024-02-20T22:48:08.810Z 19 380-399 154 154 154 265 265 265 F F F
169102024-02-20T22:48:08.810Z 20 400-419 155 155 155 266 266 266 F F F
169112024-02-20T22:48:08.810Z 21 420-439 155 156 156 266 268 268 F F F <---
169122024-02-20T22:48:08.810Z 22 440-459 155 155 155 266 266 266 F F F
169132024-02-20T22:48:08.810Z 23 460-479 155 155 155 266 266 266 F F F
169142024-02-20T22:48:08.810Z 24 480-499 155 155 155 266 266 266 F F F
169152024-02-20T22:48:08.810Z 25 500-519 152 152 152 260 260 260 F F F
169162024-02-20T22:48:08.810Z 26 520-539 151 156 156 259 268 268 F F F <---
169172024-02-20T22:48:08.810Z 27 540-559 155 156 156 266 268 268 F F F <---
169182024-02-20T22:48:08.810Z 28 560-579 150 150 150 257 257 257 F F F
169192024-02-20T22:48:08.810Z 29 580-599 145 145 145 249 249 249 F F F
169202024-02-20T22:48:08.810ZMax gen: 156, Max flush: 268
169212024-02-20T22:48:08.810ZError: Difference in extent metadata found!
169222024-02-20T22:48:08.810ZA Difference in extent metadata is expected here
169232024-02-20T22:48:08.810ZOn loop 78
169242024-02-20T22:48:08.810Z
169252024-02-20T22:48:08.815ZVerifying data now
169262024-02-20T22:48:13.283ZNo change after verify
169272024-02-20T22:48:13.283ZLoop: 78 Downstairs dump after verify (and repair):
169282024-02-20T22:48:13.297ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169292024-02-20T22:48:13.297Z 0 000-019 155 155 155 266 266 266 F F F
169302024-02-20T22:48:13.297Z 1 020-039 156 156 156 268 268 268 F F F
169312024-02-20T22:48:13.297Z 2 040-059 152 152 152 260 260 260 F F F
169322024-02-20T22:48:13.297Z 3 060-079 148 148 148 253 253 253 F F F
169332024-02-20T22:48:13.297Z 4 080-099 152 152 152 260 260 260 F F F
169342024-02-20T22:48:13.298Z 5 100-119 153 153 153 261 261 261 F F F
169352024-02-20T22:48:13.298Z 6 120-139 156 156 156 268 268 268 F F F
169362024-02-20T22:48:13.302Z 7 140-159 141 141 141 246 246 246 F F F
169372024-02-20T22:48:13.302Z 8 160-179 155 155 155 266 266 266 F F F
169382024-02-20T22:48:13.302Z 9 180-199 155 155 155 266 266 266 F F F
169392024-02-20T22:48:13.302Z 10 200-219 155 155 155 266 266 266 F F F
169402024-02-20T22:48:13.302Z 11 220-239 156 156 156 268 268 268 F F F
169412024-02-20T22:48:13.302Z 12 240-259 156 156 156 268 268 268 F F F
169422024-02-20T22:48:13.303Z 13 260-279 152 152 152 260 260 260 F F F
169432024-02-20T22:48:13.303Z 14 280-299 153 153 153 261 261 261 F F F
169442024-02-20T22:48:13.303Z 15 300-319 157 157 157 269 269 269 F F F
169452024-02-20T22:48:13.303Z 16 320-339 155 155 155 266 266 266 F F F
169462024-02-20T22:48:13.303Z 17 340-359 151 151 151 259 259 259 F F F
169472024-02-20T22:48:13.303Z 18 360-379 157 157 157 269 269 269 F F F
169482024-02-20T22:48:13.303Z 19 380-399 154 154 154 265 265 265 F F F
169492024-02-20T22:48:13.303Z 20 400-419 155 155 155 266 266 266 F F F
169502024-02-20T22:48:13.303Z 21 420-439 156 156 156 268 268 268 F F F
169512024-02-20T22:48:13.303Z 22 440-459 155 155 155 266 266 266 F F F
169522024-02-20T22:48:13.303Z 23 460-479 155 155 155 266 266 266 F F F
169532024-02-20T22:48:13.303Z 24 480-499 155 155 155 266 266 266 F F F
169542024-02-20T22:48:13.303Z 25 500-519 152 152 152 260 260 260 F F F
169552024-02-20T22:48:13.304Z 26 520-539 156 156 156 268 268 268 F F F
169562024-02-20T22:48:13.304Z 27 540-559 156 156 156 268 268 268 F F F
169572024-02-20T22:48:13.304Z 28 560-579 150 150 150 257 257 257 F F F
169582024-02-20T22:48:13.304Z 29 580-599 145 145 145 249 249 249 F F F
169592024-02-20T22:48:13.304ZMax gen: 157, Max flush: 269
169602024-02-20T22:48:13.304Z
169612024-02-20T22:48:13.304ZBegin loop 79
169622024-02-20T22:48:13.304ZDownstairs to restart: 2
169632024-02-20T22:48:13.319Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828131317Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770}
169642024-02-20T22:48:13.328Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","Crucible runtime is spawned
169652024-02-20T22:48:13.328Zlevel":30,"time":"2024-02-20T22:48:12.828552712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770}
169662024-02-20T22:48:13.328Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828586501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770}
169672024-02-20T22:48:13.328Z{"msg":"Crucible d7c5b0fd-4452-4ad1-8b7f-c6380cc7eeb5 has session id: a1b533da-8314-447a-93fe-a3a96ddced28","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.82861763Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169682024-02-20T22:48:13.328Z{"msg":"Upstairs opts: Upstairs UUID: d7c5b0fd-4452-4ad1-8b7f-c6380cc7eeb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828660468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169692024-02-20T22:48:13.328Z{"msg":"Crucible stats registered with UUID: d7c5b0fd-4452-4ad1-8b7f-c6380cc7eeb5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828701457Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169702024-02-20T22:48:13.329Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828719466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770}
169712024-02-20T22:48:13.329Z{"msg":"d7c5b0fd-4452-4ad1-8b7f-c6380cc7eeb5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828743615Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169722024-02-20T22:48:13.329Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828760145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169732024-02-20T22:48:13.329Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828796383Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169742024-02-20T22:48:13.329Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828842102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169752024-02-20T22:48:13.329Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.82888962Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169762024-02-20T22:48:13.329Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828965487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169772024-02-20T22:48:13.329Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.828998476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169782024-02-20T22:48:13.329Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829025675Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169792024-02-20T22:48:13.329Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829053514Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169802024-02-20T22:48:13.329Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829078623Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169812024-02-20T22:48:13.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829129271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169822024-02-20T22:48:13.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.82916804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169832024-02-20T22:48:13.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829206689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169842024-02-20T22:48:13.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829754109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169852024-02-20T22:48:13.329Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829801787Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169862024-02-20T22:48:13.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829844936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169872024-02-20T22:48:13.329Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829875155Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169882024-02-20T22:48:13.330Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829915623Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169892024-02-20T22:48:13.330Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.829945002Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169902024-02-20T22:48:13.330Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830360917Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169912024-02-20T22:48:13.330Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830398086Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169922024-02-20T22:48:13.330Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830431335Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169932024-02-20T22:48:13.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830876519Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169942024-02-20T22:48:13.330Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830946826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169952024-02-20T22:48:13.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830979295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169962024-02-20T22:48:13.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.830994365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169972024-02-20T22:48:13.330Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831015704Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169982024-02-20T22:48:13.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831029503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
169992024-02-20T22:48:13.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831048813Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170002024-02-20T22:48:13.330Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831062752Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170012024-02-20T22:48:13.330Z{"msg":"[0]R flush_numbers[0..12]: [266, 268, 260, 253, 260, 261, 268, 246, 266, 266, 266, 268]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831083861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170022024-02-20T22:48:13.330Z{"msg":"[0]R generation[0..12]: [155, 156, 152, 148, 152, 153, 156, 141, 155, 155, 155, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831098561Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170032024-02-20T22:48:13.331Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.83112566Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170042024-02-20T22:48:13.331Z{"msg":"[1]R flush_numbers[0..12]: [266, 268, 260, 253, 260, 261, 268, 246, 266, 266, 266, 268]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
170052024-02-20T22:48:13.331Z,"time":"2024-02-20T22:48:12.83114041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170062024-02-20T22:48:13.331Z{"msg":"[1]R generation[0..12]: [155, 156, 152, 148, 152, 153, 156, 141, 155, 155, 155, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831158739Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170072024-02-20T22:48:13.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831179908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170082024-02-20T22:48:13.331Z{"msg":"[2]R flush_numbers[0..12]: [266, 268, 260, 253, 260, 261, 268, 246, 266, 266, 266, 268]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831194348Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170092024-02-20T22:48:13.331Z{"msg":"[2]R generation[0..12]: [155, 156, 152, 148, 152, 153, 156, 141, 155, 155, 155, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831214757Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170102024-02-20T22:48:13.331Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831228996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170112024-02-20T22:48:13.331Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831249006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170122024-02-20T22:48:13.331Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831262545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170132024-02-20T22:48:13.331Z{"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831276295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170142024-02-20T22:48:13.331Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831296404Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170152024-02-20T22:48:13.331Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831310524Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170162024-02-20T22:48:13.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831322713Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170172024-02-20T22:48:13.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831343162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170182024-02-20T22:48:13.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831356902Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170192024-02-20T22:48:13.332Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831370291Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170202024-02-20T22:48:13.332Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831390861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170212024-02-20T22:48:13.332Z{"msg":"d7c5b0fd-4452-4ad1-8b7f-c6380cc7eeb5 is now active with session: a1b533da-8314-447a-93fe-a3a96ddced28","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.8314042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170222024-02-20T22:48:13.332Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831422949Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770,"session_id":"a1b533da-8314-447a-93fe-a3a96ddced28"}
170232024-02-20T22:48:13.332Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:12.831435659Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770}
170242024-02-20T22:48:13.332ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170252024-02-20T22:48:13.332ZLoading write count information from file "/tmp/test_repair_verify.data"
170262024-02-20T22:48:13.332ZRun Repair workload
170272024-02-20T22:48:13.332Z01/30 Write block 227 len 8192 data: 11 14
170282024-02-20T22:48:13.340Z02/30 Read block 76 len 16384
170292024-02-20T22:48:13.341Z03/30 Write block 525 len 28672 data: 9 11 12 10 10 12 11
170302024-02-20T22:48:13.341Z04/30 Write block 541 len 28672 data: 15 12 13 11 11 11 12
170312024-02-20T22:48:13.341Z05/30 Read block 159 len 8192
170322024-02-20T22:48:13.341Z06/30 Read block 510 len 4096
170332024-02-20T22:48:13.341Z07/30 Write block 37 len 28672 data: 18 16 15 11 10 10 11
170342024-02-20T22:48:13.341Z08/30 Read block 365 len 24576
170352024-02-20T22:48:13.341Z09/30 Read block 30 len 16384
170362024-02-20T22:48:13.341Z10/30 Write block 336 len 32768 data: 13 14 16 15 15 15 13 13
170372024-02-20T22:48:13.341Z11/30 Write block 171 len 8192 data: 9 10
170382024-02-20T22:48:13.341Z12/30 Read block 171 len 24576
170392024-02-20T22:48:13.341Z13/30 Flush
170402024-02-20T22:48:13.341Z14/30 Write block 518 len 4096 data: 12
170412024-02-20T22:48:13.342Z15/30 Read block 27 len 12288
170422024-02-20T22:48:13.342Z16/30 Write block 191 len 36864 data: 11 11 10 10 14 12 13 12 12
170432024-02-20T22:48:13.342Z17/30 Read block 146 len 24576
170442024-02-20T22:48:13.342Z18/30 Flush
170452024-02-20T22:48:13.346Z19/30 Write block 576 len 32768 data: 11 15 18 21 19 16 16 14
170462024-02-20T22:48:13.346Z20/30 Write block 517 len 8192 data: 14 13
170472024-02-20T22:48:13.346Z21/30 Read block 254 len 4096
170482024-02-20T22:48:13.347Z22/30 Write block 356 len 28672 data: 10 6 9 14 16 16 19
170492024-02-20T22:48:13.347Z23/30 Write block 461 len 12288 data: 19 19 20
170502024-02-20T22:48:13.347Z24/30 Read block 576 len 28672
170512024-02-20T22:48:13.360Z25/30 Write block 235 len 40960 data: 19 18 16 11 10 8 9 8 8 7
170522024-02-20T22:48:13.360Z26/30 Write block 458 len 32768 data: 20 19 17 20 20 21 17 16
170532024-02-20T22:48:13.360Z27/30 Write block 63 len 40960 data: 10 9 11 10 14 12 13 15 15 14
170542024-02-20T22:48:13.360Z28/30 Read block 288 len 36864
170552024-02-20T22:48:13.360Z29/30 Read block 465 len 36864
170562024-02-20T22:48:13.360Z30/30 Read block 503 len 32768
170572024-02-20T22:48:13.367Z----------------------------------------------------------------
170582024-02-20T22:48:13.367Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170592024-02-20T22:48:13.367ZStates: Active Active Active
170602024-02-20T22:48:13.367ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170612024-02-20T22:48:13.367Z 1 Acked 1000 Write 2 Done Done Sent false
170622024-02-20T22:48:13.367Z 2 Acked 1001 Read 4 Done Done Sent false
170632024-02-20T22:48:13.367Z 3 Acked 1002 Write 7 Done Done Sent false
170642024-02-20T22:48:13.367Z 4 Acked 1003 Write 7 Done Done Sent false
170652024-02-20T22:48:13.367Z 5 Acked 1004 Read 2 Done Done Sent false
170662024-02-20T22:48:13.367Z 6 Acked 1005 Read 1 Done Done Sent false
170672024-02-20T22:48:13.367Z 7 Acked 1006 Write 7 Done Done Sent false
170682024-02-20T22:48:13.367Z 8 Acked 1007 Read 6 Done Done Sent false
170692024-02-20T22:48:13.367Z 9 Acked 1008 Read 4 Done Done Sent false
170702024-02-20T22:48:13.368Z 10 Acked 1009 Write 8 Done Done Sent false
170712024-02-20T22:48:13.368Z 11 Acked 1010 Write 2 Done Done Sent false
170722024-02-20T22:48:13.368Z 12 Acked 1011 Read 6 Done Done Sent false
170732024-02-20T22:48:13.368Z 13 Acked 1012 Flush 0 Done Done Sent false
170742024-02-20T22:48:13.368Z 14 Acked 1013 Write 1 Done Done Sent false
170752024-02-20T22:48:13.368Z 15 Acked 1014 Read 3 Done Done Sent false
170762024-02-20T22:48:13.368Z 16 Acked 1015 Write 9 Done Done Sent false
170772024-02-20T22:48:13.368Z 17 Acked 1016 Read 6 Done Done Sent false
170782024-02-20T22:48:13.368Z 18 Acked 1017 Flush 0 Done Done Sent false
170792024-02-20T22:48:13.368Z 19 Acked 1018 Write 8 Done Done Sent false
170802024-02-20T22:48:13.368Z 20 Acked 1019 Write 2 Done Done Sent false
170812024-02-20T22:48:13.368Z 21 Acked 1020 Read 1 Done Done Sent false
170822024-02-20T22:48:13.368Z 22 Acked 1021 Write 7 Done Done Sent false
170832024-02-20T22:48:13.368Z 23 Acked 1022 Write 3 Done Done Sent false
170842024-02-20T22:48:13.368Z 24 Acked 1023 Read 7 Done Done Sent false
170852024-02-20T22:48:13.368Z 25 Acked 1024 Write 10 Done Done Sent false
170862024-02-20T22:48:13.368Z 26 Acked 1025 Write 8 Done Done Sent false
170872024-02-20T22:48:13.368Z 27 Acked 1026 Write 10 Done Done Sent false
170882024-02-20T22:48:13.368Z 28 Acked 1027 Read 9 Done Done Sent false
170892024-02-20T22:48:13.368Z 29 Acked 1028 Read 9 Done Done Sent false
170902024-02-20T22:48:13.368Z 30 Acked 1029 Read 8 Done Sent Sent false
170912024-02-20T22:48:13.368Z STATES DS:0 DS:1 DS:2 TOTAL
170922024-02-20T22:48:13.368Z New 0 0 0 0
170932024-02-20T22:48:13.368Z Sent 0 1 30 31
170942024-02-20T22:48:13.369Z Done 30 29 0 59
170952024-02-20T22:48:13.369Z Skipped 0 0 0 0
170962024-02-20T22:48:13.369Z Error 0 0 0 0
170972024-02-20T22:48:13.369ZLast Flush: 1017 1017 0
170982024-02-20T22:48:13.369ZDownstairs last five completed:
170992024-02-20T22:48:13.369ZUpstairs last five completed: 30 29 28 27 26
171002024-02-20T22:48:13.369Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:12.862211898Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1770}
171012024-02-20T22:48:13.369ZWrote out file "/tmp/test_repair_verify.data"
171022024-02-20T22:48:13.369Z
171032024-02-20T22:48:13.369ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171042024-02-20T22:48:13.376ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171052024-02-20T22:48:13.377Z 0 000-019 155 155 155 266 266 266 F F F
171062024-02-20T22:48:13.377Z 1 020-039 158 158 156 270 270 268 F F F <---
171072024-02-20T22:48:13.377Z 2 040-059 158 158 152 270 270 260 F F F <---
171082024-02-20T22:48:13.377Z 3 060-079 148 148 148 253 253 253 T T F <---
171092024-02-20T22:48:13.377Z 4 080-099 152 152 152 260 260 260 F F F
171102024-02-20T22:48:13.377Z 5 100-119 153 153 153 261 261 261 F F F
171112024-02-20T22:48:13.377Z 6 120-139 156 156 156 268 268 268 F F F
171122024-02-20T22:48:13.381Z 7 140-159 141 141 141 246 246 246 F F F
171132024-02-20T22:48:13.381Z 8 160-179 158 158 155 270 270 266 F F F <---
171142024-02-20T22:48:13.382Z 9 180-199 158 158 155 271 271 266 F F F <---
171152024-02-20T22:48:13.382Z 10 200-219 155 155 155 266 266 266 F F F
171162024-02-20T22:48:13.382Z 11 220-239 158 158 156 270 270 268 T T F <---
171172024-02-20T22:48:13.382Z 12 240-259 156 156 156 268 268 268 T T F <---
171182024-02-20T22:48:13.382Z 13 260-279 152 152 152 260 260 260 F F F
171192024-02-20T22:48:13.382Z 14 280-299 153 153 153 261 261 261 F F F
171202024-02-20T22:48:13.382Z 15 300-319 157 157 157 269 269 269 F F F
171212024-02-20T22:48:13.382Z 16 320-339 158 158 155 270 270 266 F F F <---
171222024-02-20T22:48:13.382Z 17 340-359 158 158 151 270 270 259 T T F <---
171232024-02-20T22:48:13.382Z 18 360-379 157 157 157 269 269 269 T T F <---
171242024-02-20T22:48:13.382Z 19 380-399 154 154 154 265 265 265 F F F
171252024-02-20T22:48:13.382Z 20 400-419 155 155 155 266 266 266 F F F
171262024-02-20T22:48:13.382Z 21 420-439 156 156 156 268 268 268 F F F
171272024-02-20T22:48:13.383Z 22 440-459 155 155 155 266 266 266 T T F <---
171282024-02-20T22:48:13.383Z 23 460-479 155 155 155 266 266 266 T T F <---
171292024-02-20T22:48:13.383Z 24 480-499 155 155 155 266 266 266 F F F
171302024-02-20T22:48:13.383Z 25 500-519 158 158 152 271 271 260 T T F <---
171312024-02-20T22:48:13.383Z 26 520-539 158 158 156 270 270 268 F F F <---
171322024-02-20T22:48:13.383Z 27 540-559 158 158 156 270 270 268 F F F <---
171332024-02-20T22:48:13.383Z 28 560-579 150 150 150 257 257 257 T T F <---
171342024-02-20T22:48:13.383Z 29 580-599 145 145 145 249 249 249 T T F <---
171352024-02-20T22:48:13.383ZMax gen: 158, Max flush: 271
171362024-02-20T22:48:13.383ZError: Difference in extent metadata found!
171372024-02-20T22:48:13.383ZA Difference in extent metadata is expected here
171382024-02-20T22:48:13.383ZOn loop 79
171392024-02-20T22:48:13.384Z
171402024-02-20T22:48:13.388ZVerifying data now
171412024-02-20T22:48:18.190ZNo change after verify
171422024-02-20T22:48:18.190ZLoop: 79 Downstairs dump after verify (and repair):
171432024-02-20T22:48:18.204ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171442024-02-20T22:48:18.204Z 0 000-019 155 155 155 266 266 266 F F F
171452024-02-20T22:48:18.204Z 1 020-039 158 158 158 270 270 270 F F F
171462024-02-20T22:48:18.204Z 2 040-059 158 158 158 270 270 270 F F F
171472024-02-20T22:48:18.204Z 3 060-079 159 159 159 272 272 272 F F F
171482024-02-20T22:48:18.204Z 4 080-099 152 152 152 260 260 260 F F F
171492024-02-20T22:48:18.204Z 5 100-119 153 153 153 261 261 261 F F F
171502024-02-20T22:48:18.209Z 6 120-139 156 156 156 268 268 268 F F F
171512024-02-20T22:48:18.209Z 7 140-159 141 141 141 246 246 246 F F F
171522024-02-20T22:48:18.209Z 8 160-179 158 158 158 270 270 270 F F F
171532024-02-20T22:48:18.209Z 9 180-199 158 158 158 271 271 271 F F F
171542024-02-20T22:48:18.209Z 10 200-219 155 155 155 266 266 266 F F F
171552024-02-20T22:48:18.209Z 11 220-239 159 159 159 272 272 272 F F F
171562024-02-20T22:48:18.209Z 12 240-259 159 159 159 272 272 272 F F F
171572024-02-20T22:48:18.209Z 13 260-279 152 152 152 260 260 260 F F F
171582024-02-20T22:48:18.209Z 14 280-299 153 153 153 261 261 261 F F F
171592024-02-20T22:48:18.209Z 15 300-319 157 157 157 269 269 269 F F F
171602024-02-20T22:48:18.209Z 16 320-339 158 158 158 270 270 270 F F F
171612024-02-20T22:48:18.209Z 17 340-359 159 159 159 272 272 272 F F F
171622024-02-20T22:48:18.209Z 18 360-379 159 159 159 272 272 272 F F F
171632024-02-20T22:48:18.209Z 19 380-399 154 154 154 265 265 265 F F F
171642024-02-20T22:48:18.210Z 20 400-419 155 155 155 266 266 266 F F F
171652024-02-20T22:48:18.210Z 21 420-439 156 156 156 268 268 268 F F F
171662024-02-20T22:48:18.210Z 22 440-459 159 159 159 272 272 272 F F F
171672024-02-20T22:48:18.210Z 23 460-479 159 159 159 272 272 272 F F F
171682024-02-20T22:48:18.210Z 24 480-499 155 155 155 266 266 266 F F F
171692024-02-20T22:48:18.210Z 25 500-519 159 159 159 272 272 272 F F F
171702024-02-20T22:48:18.210Z 26 520-539 158 158 158 270 270 270 F F F
171712024-02-20T22:48:18.210Z 27 540-559 158 158 158 270 270 270 F F F
171722024-02-20T22:48:18.210Z 28 560-579 159 159 159 272 272 272 F F F
171732024-02-20T22:48:18.210Z 29 580-599 159 159 159 272 272 272 F F F
171742024-02-20T22:48:18.210ZMax gen: 159, Max flush: 272
171752024-02-20T22:48:18.210Z
171762024-02-20T22:48:18.210ZBegin loop 80
171772024-02-20T22:48:18.210ZDownstairs to restart: 2
171782024-02-20T22:48:18.225Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.733955042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778}
171792024-02-20T22:48:18.225Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734232182Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778}
171802024-02-20T22:48:18.225Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734258331Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778}
171812024-02-20T22:48:18.235Z{"msg":"Crucible b7d0a7e8-a8b0-43e1-b75c-634f5c7536ab has session id: f8384fd1-b617-485e-97e5-8c932a77d5bc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734273101Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171822024-02-20T22:48:18.235Z{"msg":"Upstairs opts: Upstairs UUID: b7d0a7e8-a8b0-43e1-b75c-634f5c7536ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734310409Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171832024-02-20T22:48:18.235Z{"msg":"Crucible runtime is spawned
171842024-02-20T22:48:18.235ZCrucible stats registered with UUID: b7d0a7e8-a8b0-43e1-b75c-634f5c7536ab","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734345478Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171852024-02-20T22:48:18.235Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734434315Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778}
171862024-02-20T22:48:18.235Z{"msg":"b7d0a7e8-a8b0-43e1-b75c-634f5c7536ab active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734543701Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171872024-02-20T22:48:18.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.73457962Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171882024-02-20T22:48:18.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734600699Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171892024-02-20T22:48:18.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734636388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171902024-02-20T22:48:18.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734679476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171912024-02-20T22:48:18.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734719905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171922024-02-20T22:48:18.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734750304Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171932024-02-20T22:48:18.235Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734778712Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171942024-02-20T22:48:18.235Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734837321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171952024-02-20T22:48:18.235Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734863709Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171962024-02-20T22:48:18.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734889688Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171972024-02-20T22:48:18.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734965426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171982024-02-20T22:48:18.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.734991495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
171992024-02-20T22:48:18.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.735564344Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172002024-02-20T22:48:18.235Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.735599783Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172012024-02-20T22:48:18.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.735615563Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172022024-02-20T22:48:18.235Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.735629602Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172032024-02-20T22:48:18.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.735654881Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172042024-02-20T22:48:18.236Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.73567196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172052024-02-20T22:48:18.236Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736192892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172062024-02-20T22:48:18.236Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736231551Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172072024-02-20T22:48:18.236Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736293028Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172082024-02-20T22:48:18.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736789351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172092024-02-20T22:48:18.236Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736826049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172102024-02-20T22:48:18.236Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736841049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172112024-02-20T22:48:18.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736921736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172122024-02-20T22:48:18.236Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736950105Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172132024-02-20T22:48:18.236Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736964474Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172142024-02-20T22:48:18.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.736980844Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172152024-02-20T22:48:18.236Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737011823Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172162024-02-20T22:48:18.236Z{"msg":"[0]R flush_numbers[0..12]: [266, 270, 270, 272, 260, 261, 268, 246, 270, 271, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737028352Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172172024-02-20T22:48:18.236Z{"msg":"[0]R generation[0..12]: [155, 158, 158, 159, 152, 153, 156, 141, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737046211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172182024-02-20T22:48:18.236Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.73707686Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172192024-02-20T22:48:18.236Z{"msg":"[1]R flush_numbers[0..12]: [266, 270, 270, 272, 260, 261, 268, 246, 270, 271, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.73709329Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172202024-02-20T22:48:18.236Z{"msg":"[1]R generation[0..12]: [155, 158, 158, 159, 152, 153, 156, 141, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737125399Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172212024-02-20T22:48:18.236Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737142558Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,""Wait for a query_work_queue command to finish before sending IO
172222024-02-20T22:48:18.236Z:"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172232024-02-20T22:48:18.236Z{"msg":"[2]R flush_numbers[0..12]: [266, 270, 270, 272, 260, 261, 268, 246, 270, 271, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737175557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172242024-02-20T22:48:18.236Z{"msg":"[2]R generation[0..12]: [155, 158, 158, 159, 152, 153, 156, 141, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737192716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172252024-02-20T22:48:18.236Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737223845Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172262024-02-20T22:48:18.236Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737238924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172272024-02-20T22:48:18.236Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737254674Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172282024-02-20T22:48:18.236Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737284263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172292024-02-20T22:48:18.237Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737299482Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172302024-02-20T22:48:18.237Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737330261Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172312024-02-20T22:48:18.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737345271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172322024-02-20T22:48:18.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.73737628Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172332024-02-20T22:48:18.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737392589Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172342024-02-20T22:48:18.237Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737422268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172352024-02-20T22:48:18.237Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737436477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172362024-02-20T22:48:18.237Z{"msg":"b7d0a7e8-a8b0-43e1-b75c-634f5c7536ab is now active with session: f8384fd1-b617-485e-97e5-8c932a77d5bc","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737455797Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172372024-02-20T22:48:18.237Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737482866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778,"session_id":"f8384fd1-b617-485e-97e5-8c932a77d5bc"}
172382024-02-20T22:48:18.237Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:17.737496825Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778}
172392024-02-20T22:48:18.237ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172402024-02-20T22:48:18.237ZLoading write count information from file "/tmp/test_repair_verify.data"
172412024-02-20T22:48:18.237ZRun Repair workload
172422024-02-20T22:48:18.237Z01/30 Write block 529 len 28672 data: 11 13 12 8 10 13 16
172432024-02-20T22:48:18.242Z02/30 Write block 63 len 4096 data: 11
172442024-02-20T22:48:18.242Z03/30 Write block 249 len 32768 data: 8 10 11 11 12 11 10 9
172452024-02-20T22:48:18.242Z04/30 Write block 531 len 32768 data: 13 9 11 14 17 16 14 18
172462024-02-20T22:48:18.242Z05/30 Write block 9 len 32768 data: 8 11 9 8 10 9 10 14
172472024-02-20T22:48:18.242Z06/30 Read block 45 len 36864
172482024-02-20T22:48:18.242Z07/30 Write block 343 len 4096 data: 14
172492024-02-20T22:48:18.242Z08/30 Read block 512 len 32768
172502024-02-20T22:48:18.242Z09/30 Write block 467 len 12288 data: 14 17 20
172512024-02-20T22:48:18.242Z10/30 Read block 287 len 12288
172522024-02-20T22:48:18.242Z11/30 Read block 61 len 32768
172532024-02-20T22:48:18.242Z12/30 Write block 569 len 12288 data: 6 7 5
172542024-02-20T22:48:18.242Z13/30 Write block 25 len 32768 data: 15 13 18 18 23 23 24 25
172552024-02-20T22:48:18.242Z14/30 Write block 275 len 32768 data: 10 8 9 6 6 7 9 8
172562024-02-20T22:48:18.242Z15/30 Write block 53 len 4096 data: 9
172572024-02-20T22:48:18.242Z16/30 Read block 432 len 32768
172582024-02-20T22:48:18.247Z17/30 Read block 257 len 32768
172592024-02-20T22:48:18.247Z18/30 Read block 456 len 32768
172602024-02-20T22:48:18.247Z19/30 Read block 2 len 16384
172612024-02-20T22:48:18.247Z20/30 Flush
172622024-02-20T22:48:18.250Z21/30 Write block 60 len 16384 data: 12 10 10 12
172632024-02-20T22:48:18.255Z22/30 Write block 404 len 28672 data: 15 14 14 12 10 12 13
172642024-02-20T22:48:18.255Z23/30 Write block 455 len 16384 data: 12 18 18 21
172652024-02-20T22:48:18.255Z24/30 Flush
172662024-02-20T22:48:18.260Z25/30 Flush
172672024-02-20T22:48:18.260Z26/30 Write block 121 len 28672 data: 14 16 14 14 10 9 11
172682024-02-20T22:48:18.260Z27/30 Write block 61 len 36864 data: 11 11 13 10 12 11 15 13 14
172692024-02-20T22:48:18.260Z28/30 Read block 541 len 40960
172702024-02-20T22:48:18.260Z29/30 Read block 3 len 28672
172712024-02-20T22:48:18.266Z30/30 Read block 462 len 16384
172722024-02-20T22:48:18.266Z----------------------------------------------------------------
172732024-02-20T22:48:18.266Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172742024-02-20T22:48:18.267ZStates: Active Active Active
172752024-02-20T22:48:18.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172762024-02-20T22:48:18.267Z 1 Acked 1000 Write 7 Done Done Done false
172772024-02-20T22:48:18.267Z 2 Acked 1001 Write 1 Done Done Done false
172782024-02-20T22:48:18.267Z 3 Acked 1002 Write 8 Done Done Done false
172792024-02-20T22:48:18.267Z 4 Acked 1003 Write 8 Done Done Done false
172802024-02-20T22:48:18.267Z 5 Acked 1004 Write 8 Done Done Done false
172812024-02-20T22:48:18.267Z 6 Acked 1005 Read 9 Done Done Done false
172822024-02-20T22:48:18.267Z 7 Acked 1006 Write 1 Done Done Done false
172832024-02-20T22:48:18.267Z 8 Acked 1007 Read 8 Done Done Done false
172842024-02-20T22:48:18.267Z 9 Acked 1008 Write 3 Done Done Done false
172852024-02-20T22:48:18.267Z 10 Acked 1009 Read 3 Done Done Sent false
172862024-02-20T22:48:18.267Z 11 Acked 1010 Read 8 Done Done Sent false
172872024-02-20T22:48:18.267Z 12 Acked 1011 Write 3 Done Done Sent false
172882024-02-20T22:48:18.267Z 13 Acked 1012 Write 8 Done Done Sent false
172892024-02-20T22:48:18.267Z 14 Acked 1013 Write 8 Done Done Sent false
172902024-02-20T22:48:18.267Z 15 Acked 1014 Write 1 Done Done Sent false
172912024-02-20T22:48:18.267Z 16 Acked 1015 Read 8 Done Done Sent false
172922024-02-20T22:48:18.267Z 17 Acked 1016 Read 8 Done Done Sent false
172932024-02-20T22:48:18.267Z 18 Acked 1017 Read 8 Done Done Sent false
172942024-02-20T22:48:18.267Z 19 Acked 1018 Read 4 Done Done Sent false
172952024-02-20T22:48:18.267Z 20 Acked 1019 Flush 0 Done Done Sent false
172962024-02-20T22:48:18.267Z 21 Acked 1020 Write 4 Done Done Sent false
172972024-02-20T22:48:18.267Z 22 Acked 1021 Write 7 Done Done Sent false
172982024-02-20T22:48:18.267Z 23 Acked 1022 Write 4 Done Done Sent false
172992024-02-20T22:48:18.268Z 24 Acked 1023 Flush 0 Done Done Sent false
173002024-02-20T22:48:18.268Z 25 Acked 1024 Flush 0 Done Done Sent false
173012024-02-20T22:48:18.268Z 26 Acked 1025 Write 7 Done Done Sent false
173022024-02-20T22:48:18.268Z 27 Acked 1026 Write 9 Done Done Sent false
173032024-02-20T22:48:18.268Z 28 Acked 1027 Read 10 Done Done Sent false
173042024-02-20T22:48:18.268Z 29 Acked 1028 Read 7 Done Done Sent false
173052024-02-20T22:48:18.268Z 30 Acked 1029 Read 4 Done Done Sent false
173062024-02-20T22:48:18.268Z STATES DS:0 DS:1 DS:2 TOTAL
173072024-02-20T22:48:18.268Z New 0 0 0 0
173082024-02-20T22:48:18.268Z Sent 0 0 21 21
173092024-02-20T22:48:18.268Z Done 30 30 9 69
173102024-02-20T22:48:18.268Z Skipped 0 0 0 0
173112024-02-20T22:48:18.268Z Error 0 0 0 0
173122024-02-20T22:48:18.268ZLast Flush: 1024 1024 0
173132024-02-20T22:48:18.268ZDownstairs last five completed:
173142024-02-20T22:48:18.268ZUpstairs last five completed: 30 29 28 27 26
173152024-02-20T22:48:18.268Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:17.770412268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1778}
173162024-02-20T22:48:18.268ZWrote out file "/tmp/test_repair_verify.data"
173172024-02-20T22:48:18.272Z
173182024-02-20T22:48:18.273ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173192024-02-20T22:48:18.284ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173202024-02-20T22:48:18.284Z 0 000-019 160 160 155 273 273 266 F F T <---
173212024-02-20T22:48:18.284Z 1 020-039 160 160 158 273 273 270 F F F <---
173222024-02-20T22:48:18.285Z 2 040-059 160 160 158 273 273 270 F F F <---
173232024-02-20T22:48:18.285Z 3 060-079 160 160 159 274 274 272 T T T <---
173242024-02-20T22:48:18.285Z 4 080-099 152 152 152 260 260 260 F F F
173252024-02-20T22:48:18.285Z 5 100-119 153 153 153 261 261 261 F F F
173262024-02-20T22:48:18.285Z 6 120-139 156 156 156 268 268 268 T T F <---
173272024-02-20T22:48:18.285Z 7 140-159 141 141 141 246 246 246 F F F
173282024-02-20T22:48:18.285Z 8 160-179 158 158 158 270 270 270 F F F
173292024-02-20T22:48:18.290Z 9 180-199 158 158 158 271 271 271 F F F
173302024-02-20T22:48:18.290Z 10 200-219 155 155 155 266 266 266 F F F
173312024-02-20T22:48:18.290Z 11 220-239 159 159 159 272 272 272 F F F
173322024-02-20T22:48:18.290Z 12 240-259 160 160 159 273 273 272 F F T <---
173332024-02-20T22:48:18.290Z 13 260-279 160 160 152 273 273 260 F F F <---
173342024-02-20T22:48:18.290Z 14 280-299 160 160 153 273 273 261 F F F <---
173352024-02-20T22:48:18.290Z 15 300-319 157 157 157 269 269 269 F F F
173362024-02-20T22:48:18.290Z 16 320-339 158 158 158 270 270 270 F F F
173372024-02-20T22:48:18.290Z 17 340-359 160 160 159 273 273 272 F F T <---
173382024-02-20T22:48:18.290Z 18 360-379 159 159 159 272 272 272 F F F
173392024-02-20T22:48:18.290Z 19 380-399 154 154 154 265 265 265 F F F
173402024-02-20T22:48:18.290Z 20 400-419 160 160 155 274 274 266 F F F <---
173412024-02-20T22:48:18.290Z 21 420-439 156 156 156 268 268 268 F F F
173422024-02-20T22:48:18.290Z 22 440-459 160 160 159 274 274 272 F F F <---
173432024-02-20T22:48:18.290Z 23 460-479 160 160 159 273 273 272 F F T <---
173442024-02-20T22:48:18.290Z 24 480-499 155 155 155 266 266 266 F F F
173452024-02-20T22:48:18.290Z 25 500-519 159 159 159 272 272 272 F F F
173462024-02-20T22:48:18.290Z 26 520-539 160 160 158 273 273 270 F F T <---
173472024-02-20T22:48:18.290Z 27 540-559 158 158 158 270 270 270 F F F
173482024-02-20T22:48:18.290Z 28 560-579 160 160 159 273 273 272 F F F <---
173492024-02-20T22:48:18.291Z 29 580-599 159 159 159 272 272 272 F F F
173502024-02-20T22:48:18.291ZMax gen: 160, Max flush: 274
173512024-02-20T22:48:18.291ZError: Difference in extent metadata found!
173522024-02-20T22:48:18.291ZA Difference in extent metadata is expected here
173532024-02-20T22:48:18.291ZOn loop 80
173542024-02-20T22:48:18.291Z
173552024-02-20T22:48:18.295ZVerifying data now
173562024-02-20T22:48:23.966ZNo change after verify
173572024-02-20T22:48:23.970ZLoop: 80 Downstairs dump after verify (and repair):
173582024-02-20T22:48:23.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173592024-02-20T22:48:23.979Z 0 000-019 160 160 160 273 273 273 F F F
173602024-02-20T22:48:23.979Z 1 020-039 160 160 160 273 273 273 F F F
173612024-02-20T22:48:23.979Z 2 040-059 160 160 160 273 273 273 F F F
173622024-02-20T22:48:23.980Z 3 060-079 161 161 161 275 275 275 F F F
173632024-02-20T22:48:23.980Z 4 080-099 152 152 152 260 260 260 F F F
173642024-02-20T22:48:23.980Z 5 100-119 153 153 153 261 261 261 F F F
173652024-02-20T22:48:23.980Z 6 120-139 161 161 161 275 275 275 F F F
173662024-02-20T22:48:23.980Z 7 140-159 141 141 141 246 246 246 F F F
173672024-02-20T22:48:23.980Z 8 160-179 158 158 158 270 270 270 F F F
173682024-02-20T22:48:23.980Z 9 180-199 158 158 158 271 271 271 F F F
173692024-02-20T22:48:23.984Z 10 200-219 155 155 155 266 266 266 F F F
173702024-02-20T22:48:23.984Z 11 220-239 159 159 159 272 272 272 F F F
173712024-02-20T22:48:23.984Z 12 240-259 160 160 160 273 273 273 F F F
173722024-02-20T22:48:23.985Z 13 260-279 160 160 160 273 273 273 F F F
173732024-02-20T22:48:23.985Z 14 280-299 160 160 160 273 273 273 F F F
173742024-02-20T22:48:23.985Z 15 300-319 157 157 157 269 269 269 F F F
173752024-02-20T22:48:23.985Z 16 320-339 158 158 158 270 270 270 F F F
173762024-02-20T22:48:23.985Z 17 340-359 160 160 160 273 273 273 F F F
173772024-02-20T22:48:23.985Z 18 360-379 159 159 159 272 272 272 F F F
173782024-02-20T22:48:23.985Z 19 380-399 154 154 154 265 265 265 F F F
173792024-02-20T22:48:23.985Z 20 400-419 160 160 160 274 274 274 F F F
173802024-02-20T22:48:23.985Z 21 420-439 156 156 156 268 268 268 F F F
173812024-02-20T22:48:23.985Z 22 440-459 160 160 160 274 274 274 F F F
173822024-02-20T22:48:23.985Z 23 460-479 160 160 160 273 273 273 F F F
173832024-02-20T22:48:23.985Z 24 480-499 155 155 155 266 266 266 F F F
173842024-02-20T22:48:23.985Z 25 500-519 159 159 159 272 272 272 F F F
173852024-02-20T22:48:23.985Z 26 520-539 160 160 160 273 273 273 F F F
173862024-02-20T22:48:23.985Z 27 540-559 158 158 158 270 270 270 F F F
173872024-02-20T22:48:23.985Z 28 560-579 160 160 160 273 273 273 F F F
173882024-02-20T22:48:23.985Z 29 580-599 159 159 159 272 272 272 F F F
173892024-02-20T22:48:23.985ZMax gen: 161, Max flush: 275
173902024-02-20T22:48:23.985Z
173912024-02-20T22:48:23.985ZBegin loop 81
173922024-02-20T22:48:23.985ZDownstairs to restart: 0
173932024-02-20T22:48:24.000Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509067572Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786}
173942024-02-20T22:48:24.000Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509294183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786}
173952024-02-20T22:48:24.009Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509311733Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786}
173962024-02-20T22:48:24.010Z{"msg":"Crucible 226fba30-6f85-42b5-badd-a994eaa6ff23 has session id: aebb3cb2-8db8-40a3-99de-994ddaf5143e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509343882Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
173972024-02-20T22:48:24.010Z{"msg":"Upstairs opts: Upstairs UUID: 226fba30-6f85-42b5-badd-a994eaa6ff23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.50937976Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","Crucible runtime is spawned
173982024-02-20T22:48:24.010Zpid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
173992024-02-20T22:48:24.010Z{"msg":"Crucible stats registered with UUID: 226fba30-6f85-42b5-badd-a994eaa6ff23","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509421029Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174002024-02-20T22:48:24.010Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509500196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786}
174012024-02-20T22:48:24.010Z{"msg":"226fba30-6f85-42b5-badd-a994eaa6ff23 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509718668Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174022024-02-20T22:48:24.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509762807Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174032024-02-20T22:48:24.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509804085Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174042024-02-20T22:48:24.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509852123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174052024-02-20T22:48:24.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509879562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174062024-02-20T22:48:24.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509914321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174072024-02-20T22:48:24.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.5099421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174082024-02-20T22:48:24.011Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.509982089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174092024-02-20T22:48:24.011Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510026947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174102024-02-20T22:48:24.011Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510069076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174112024-02-20T22:48:24.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510100165Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174122024-02-20T22:48:24.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510135203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174132024-02-20T22:48:24.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510161772Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174142024-02-20T22:48:24.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510808819Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174152024-02-20T22:48:24.011Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510846118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174162024-02-20T22:48:24.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510870497Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174172024-02-20T22:48:24.011Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510898266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174182024-02-20T22:48:24.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.510955284Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174192024-02-20T22:48:24.011Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.511019082Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174202024-02-20T22:48:24.011Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.511445417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174212024-02-20T22:48:24.012Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.511511244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174222024-02-20T22:48:24.012Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.511546423Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174232024-02-20T22:48:24.012Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512010796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174242024-02-20T22:48:24.012Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512048455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174252024-02-20T22:48:24.012Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512064574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174262024-02-20T22:48:24.012Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512077994Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174272024-02-20T22:48:24.012Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512106073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174282024-02-20T22:48:24.012Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512121232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174292024-02-20T22:48:24.012Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512133692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174302024-02-20T22:48:24.012Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512147541Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174312024-02-20T22:48:24.012Z{"msg":"[0]R flush_numbers[0..12]: [273, 273, 273, 275, 260, 261, 275, 246, 270, 271, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512161581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174322024-02-20T22:48:24.012Z{"msg":"[0]R generation[0..12]: [160, 160, 160, 161, 152, 153, 161, 141, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.51217651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174332024-02-20T22:48:24.012Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.51219099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174342024-02-20T22:48:24.012Z{"msg":"[1]R flush_numbers[0..12]: [273, 273, 273, 275, 260, 261, 275, 246, 270, 271, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512212689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174352024-02-20T22:48:24.013Z{"msg":"[1]R generation[0..12]: [160, 160, 160, 161, 152, 153, 161, 141, 158, 158, 155, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512227249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174362024-02-20T22:48:24.013Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512248828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174372024-02-20T22:48:24.013Z{"msg":"[2]R flush_numbers[0..12]: [273, 273, 273, 275, 260, 261, 275, 246, 270, 271, 266, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512263437Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174382024-02-20T22:48:24.013Z{"msg":"[2]R generation[0..12]: [160, 160, 160, 161, 152, 153, 161, 141, 158, 158, 155, 159]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
174392024-02-20T22:48:24.013Z,"level":30,"time":"2024-02-20T22:48:23.512291086Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174402024-02-20T22:48:24.013Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512311955Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174412024-02-20T22:48:24.013Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512333715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174422024-02-20T22:48:24.013Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512348014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174432024-02-20T22:48:24.013Z{"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512362534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174442024-02-20T22:48:24.013Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512386663Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174452024-02-20T22:48:24.013Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512404932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174462024-02-20T22:48:24.013Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512417862Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174472024-02-20T22:48:24.013Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512441341Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174482024-02-20T22:48:24.013Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.51245747Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174492024-02-20T22:48:24.014Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512479249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174502024-02-20T22:48:24.014Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512492759Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174512024-02-20T22:48:24.014Z{"msg":"226fba30-6f85-42b5-badd-a994eaa6ff23 is now active with session: aebb3cb2-8db8-40a3-99de-994ddaf5143e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512506108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174522024-02-20T22:48:24.014Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512525108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786,"session_id":"aebb3cb2-8db8-40a3-99de-994ddaf5143e"}
174532024-02-20T22:48:24.014Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:23.512537907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786}
174542024-02-20T22:48:24.014ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174552024-02-20T22:48:24.014ZLoading write count information from file "/tmp/test_repair_verify.data"
174562024-02-20T22:48:24.014ZRun Repair workload
174572024-02-20T22:48:24.019Z01/30 Write block 238 len 24576 data: 12 11 9 10 9 9
174582024-02-20T22:48:24.019Z02/30 Read block 576 len 8192
174592024-02-20T22:48:24.019Z03/30 Write block 301 len 28672 data: 17 19 19 18 18 14 15
174602024-02-20T22:48:24.019Z04/30 Write block 477 len 36864 data: 18 20 16 19 18 18 18 16 15
174612024-02-20T22:48:24.019Z05/30 Read block 332 len 12288
174622024-02-20T22:48:24.019Z06/30 Read block 452 len 28672
174632024-02-20T22:48:24.019Z07/30 Read block 135 len 28672
174642024-02-20T22:48:24.019Z08/30 Flush
174652024-02-20T22:48:24.019Z09/30 Write block 297 len 12288 data: 9 12 13
174662024-02-20T22:48:24.019Z10/30 Write block 556 len 12288 data: 10 9 9
174672024-02-20T22:48:24.019Z11/30 Read block 544 len 24576
174682024-02-20T22:48:24.019Z12/30 Write block 553 len 36864 data: 7 7 8 11 10 10 7 6 9
174692024-02-20T22:48:24.024Z13/30 Flush
174702024-02-20T22:48:24.024Z14/30 Write block 118 len 40960 data: 9 13 13 15 17 15 15 11 10 12
174712024-02-20T22:48:24.024Z15/30 Read block 558 len 32768
174722024-02-20T22:48:24.029Z16/30 Write block 340 len 8192 data: 16 16
174732024-02-20T22:48:24.029Z17/30 Read block 256 len 8192
174742024-02-20T22:48:24.029Z18/30 Write block 277 len 40960 data: 10 7 7 8 10 9 11 11 12 11
174752024-02-20T22:48:24.029Z19/30 Write block 321 len 36864 data: 15 18 14 12 17 18 16 15 13
174762024-02-20T22:48:24.033Z20/30 Write block 487 len 16384 data: 12 12 10 10
174772024-02-20T22:48:24.033Z21/30 Read block 264 len 32768
174782024-02-20T22:48:24.033Z22/30 Read block 6 len 8192
174792024-02-20T22:48:24.033Z23/30 Write block 391 len 24576 data: 12 14 13 11 9 13
174802024-02-20T22:48:24.033Z24/30 Write block 30 len 4096 data: 24
174812024-02-20T22:48:24.033Z25/30 Read block 588 len 24576
174822024-02-20T22:48:24.040Z26/30 Write block 439 len 40960 data: 11 14 16 13 10 9 12 12 13 11
174832024-02-20T22:48:24.040Z27/30 Write block 3 len 16384 data: 11 13 13 8
174842024-02-20T22:48:24.040Z28/30 Read block 543 len 20480
174852024-02-20T22:48:24.040Z29/30 Read block 195 len 28672
174862024-02-20T22:48:24.040Z30/30 Read block 256 len 12288
174872024-02-20T22:48:24.040Z----------------------------------------------------------------
174882024-02-20T22:48:24.040Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
174892024-02-20T22:48:24.041ZStates: Active Active Active
174902024-02-20T22:48:24.041ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174912024-02-20T22:48:24.041Z 1 Acked 1000 Write 6 Sent Done Done false
174922024-02-20T22:48:24.041Z 2 Acked 1001 Read 2 Sent Done Done false
174932024-02-20T22:48:24.041Z 3 Acked 1002 Write 7 Sent Done Done false
174942024-02-20T22:48:24.041Z 4 Acked 1003 Write 9 Sent Done Done false
174952024-02-20T22:48:24.041Z 5 Acked 1004 Read 3 Sent Done Done false
174962024-02-20T22:48:24.041Z 6 Acked 1005 Read 7 Sent Done Done false
174972024-02-20T22:48:24.041Z 7 Acked 1006 Read 7 Sent Done Done false
174982024-02-20T22:48:24.041Z 8 Acked 1007 Flush 0 Sent Done Done false
174992024-02-20T22:48:24.041Z 9 Acked 1008 Write 3 Sent Done Done false
175002024-02-20T22:48:24.041Z 10 Acked 1009 Write 3 Sent Done Done false
175012024-02-20T22:48:24.041Z 11 Acked 1010 Read 6 Sent Done Done false
175022024-02-20T22:48:24.041Z 12 Acked 1011 Write 9 Sent Done Done false
175032024-02-20T22:48:24.041Z 13 Acked 1012 Flush 0 Sent Done Done false
175042024-02-20T22:48:24.041Z 14 Acked 1013 Write 10 Sent Done Done false
175052024-02-20T22:48:24.041Z 15 Acked 1014 Read 8 Sent Done Done false
175062024-02-20T22:48:24.041Z 16 Acked 1015 Write 2 Sent Done Done false
175072024-02-20T22:48:24.041Z 17 Acked 1016 Read 2 Sent Done Done false
175082024-02-20T22:48:24.041Z 18 Acked 1017 Write 10 Sent Done Done false
175092024-02-20T22:48:24.041Z 19 Acked 1018 Write 9 Sent Done Done false
175102024-02-20T22:48:24.042Z 20 Acked 1019 Write 4 Sent Done Done false
175112024-02-20T22:48:24.042Z 21 Acked 1020 Read 8 Sent Done Done false
175122024-02-20T22:48:24.042Z 22 Acked 1021 Read 2 Sent Done Done false
175132024-02-20T22:48:24.042Z 23 Acked 1022 Write 6 Sent Done Done false
175142024-02-20T22:48:24.042Z 24 Acked 1023 Write 1 Sent Done Done false
175152024-02-20T22:48:24.042Z 25 Acked 1024 Read 6 Sent Done Done false
175162024-02-20T22:48:24.042Z 26 Acked 1025 Write 10 Sent Done Done false
175172024-02-20T22:48:24.042Z 27 Acked 1026 Write 4 Sent Done Done false
175182024-02-20T22:48:24.042Z 28 Acked 1027 Read 5 Sent Done Done false
175192024-02-20T22:48:24.042Z 29 Acked 1028 Read 7 Sent Done Done false
175202024-02-20T22:48:24.042Z 30 Acked 1029 Read 3 Sent Done Done false
175212024-02-20T22:48:24.042Z STATES DS:0 DS:1 DS:2 TOTAL
175222024-02-20T22:48:24.042Z New 0 0 0 0
175232024-02-20T22:48:24.042Z Sent 30 0 0 30
175242024-02-20T22:48:24.042Z Done 0 30 30 60
175252024-02-20T22:48:24.042Z Skipped 0 0 0 0
175262024-02-20T22:48:24.042Z Error 0 0 0 0
175272024-02-20T22:48:24.042ZLast Flush: 0 1012 1012
175282024-02-20T22:48:24.042ZDownstairs last five completed:
175292024-02-20T22:48:24.042ZUpstairs last five completed: 30 29 28 27 26
175302024-02-20T22:48:24.042Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:23.546914427Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1786}
175312024-02-20T22:48:24.043ZWrote out file "/tmp/test_repair_verify.data"
175322024-02-20T22:48:24.049Z
175332024-02-20T22:48:24.049ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175342024-02-20T22:48:24.064ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175352024-02-20T22:48:24.064Z 0 000-019 160 160 160 273 273 273 F T T <---
175362024-02-20T22:48:24.064Z 1 020-039 160 160 160 273 273 273 F T T <---
175372024-02-20T22:48:24.064Z 2 040-059 160 160 160 273 273 273 F F F
175382024-02-20T22:48:24.064Z 3 060-079 161 161 161 275 275 275 F F F
175392024-02-20T22:48:24.064Z 4 080-099 152 152 152 260 260 260 F F F
175402024-02-20T22:48:24.064Z 5 100-119 153 153 153 261 261 261 F T T <---
175412024-02-20T22:48:24.064Z 6 120-139 161 161 161 275 275 275 F T T <---
175422024-02-20T22:48:24.064Z 7 140-159 141 141 141 246 246 246 F F F
175432024-02-20T22:48:24.064Z 8 160-179 158 158 158 270 270 270 F F F
175442024-02-20T22:48:24.073Z 9 180-199 158 158 158 271 271 271 F F F
175452024-02-20T22:48:24.073Z 10 200-219 155 155 155 266 266 266 F F F
175462024-02-20T22:48:24.073Z 11 220-239 159 162 162 272 276 276 F F F <---
175472024-02-20T22:48:24.073Z 12 240-259 160 162 162 273 276 276 F F F <---
175482024-02-20T22:48:24.073Z 13 260-279 160 160 160 273 273 273 F T T <---
175492024-02-20T22:48:24.073Z 14 280-299 160 162 162 273 277 277 F T T <---
175502024-02-20T22:48:24.073Z 15 300-319 157 162 162 269 276 276 F F F <---
175512024-02-20T22:48:24.073Z 16 320-339 158 158 158 270 270 270 F T T <---
175522024-02-20T22:48:24.073Z 17 340-359 160 160 160 273 273 273 F T T <---
175532024-02-20T22:48:24.073Z 18 360-379 159 159 159 272 272 272 F F F
175542024-02-20T22:48:24.074Z 19 380-399 154 154 154 265 265 265 F T T <---
175552024-02-20T22:48:24.074Z 20 400-419 160 160 160 274 274 274 F F F
175562024-02-20T22:48:24.074Z 21 420-439 156 156 156 268 268 268 F T T <---
175572024-02-20T22:48:24.074Z 22 440-459 160 160 160 274 274 274 F T T <---
175582024-02-20T22:48:24.074Z 23 460-479 160 162 162 273 276 276 F F F <---
175592024-02-20T22:48:24.074Z 24 480-499 155 162 162 266 276 276 F T T <---
175602024-02-20T22:48:24.074Z 25 500-519 159 159 159 272 272 272 F F F
175612024-02-20T22:48:24.074Z 26 520-539 160 160 160 273 273 273 F F F
175622024-02-20T22:48:24.074Z 27 540-559 158 162 162 270 277 277 F F F <---
175632024-02-20T22:48:24.074Z 28 560-579 160 162 162 273 277 277 F F F <---
175642024-02-20T22:48:24.074Z 29 580-599 159 159 159 272 272 272 F F F
175652024-02-20T22:48:24.074ZMax gen: 162, Max flush: 277
175662024-02-20T22:48:24.074ZError: Difference in extent metadata found!
175672024-02-20T22:48:24.074ZA Difference in extent metadata is expected here
175682024-02-20T22:48:24.074ZOn loop 81
175692024-02-20T22:48:24.074Z
175702024-02-20T22:48:24.079ZVerifying data now
175712024-02-20T22:48:28.943ZNo change after verify
175722024-02-20T22:48:28.943ZLoop: 81 Downstairs dump after verify (and repair):
175732024-02-20T22:48:28.965ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175742024-02-20T22:48:28.966Z 0 000-019 163 163 163 278 278 278 F F F
175752024-02-20T22:48:28.966Z 1 020-039 163 163 163 278 278 278 F F F
175762024-02-20T22:48:28.966Z 2 040-059 160 160 160 273 273 273 F F F
175772024-02-20T22:48:28.966Z 3 060-079 161 161 161 275 275 275 F F F
175782024-02-20T22:48:28.966Z 4 080-099 152 152 152 260 260 260 F F F
175792024-02-20T22:48:28.966Z 5 100-119 163 163 163 278 278 278 F F F
175802024-02-20T22:48:28.966Z 6 120-139 163 163 163 278 278 278 F F F
175812024-02-20T22:48:28.966Z 7 140-159 141 141 141 246 246 246 F F F
175822024-02-20T22:48:28.966Z 8 160-179 158 158 158 270 270 270 F F F
175832024-02-20T22:48:28.966Z 9 180-199 158 158 158 271 271 271 F F F
175842024-02-20T22:48:28.966Z 10 200-219 155 155 155 266 266 266 F F F
175852024-02-20T22:48:28.966Z 11 220-239 162 162 162 276 276 276 F F F
175862024-02-20T22:48:28.966Z 12 240-259 162 162 162 276 276 276 F F F
175872024-02-20T22:48:28.966Z 13 260-279 163 163 163 278 278 278 F F F
175882024-02-20T22:48:28.966Z 14 280-299 163 163 163 278 278 278 F F F
175892024-02-20T22:48:28.966Z 15 300-319 162 162 162 276 276 276 F F F
175902024-02-20T22:48:28.966Z 16 320-339 163 163 163 278 278 278 F F F
175912024-02-20T22:48:28.966Z 17 340-359 163 163 163 278 278 278 F F F
175922024-02-20T22:48:28.966Z 18 360-379 159 159 159 272 272 272 F F F
175932024-02-20T22:48:28.966Z 19 380-399 163 163 163 278 278 278 F F F
175942024-02-20T22:48:28.966Z 20 400-419 160 160 160 274 274 274 F F F
175952024-02-20T22:48:28.966Z 21 420-439 163 163 163 278 278 278 F F F
175962024-02-20T22:48:28.966Z 22 440-459 163 163 163 278 278 278 F F F
175972024-02-20T22:48:28.967Z 23 460-479 162 162 162 276 276 276 F F F
175982024-02-20T22:48:28.967Z 24 480-499 163 163 163 278 278 278 F F F
175992024-02-20T22:48:28.967Z 25 500-519 159 159 159 272 272 272 F F F
176002024-02-20T22:48:28.967Z 26 520-539 160 160 160 273 273 273 F F F
176012024-02-20T22:48:28.967Z 27 540-559 162 162 162 277 277 277 F F F
176022024-02-20T22:48:28.967Z 28 560-579 162 162 162 277 277 277 F F F
176032024-02-20T22:48:28.967Z 29 580-599 159 159 159 272 272 272 F F F
176042024-02-20T22:48:28.967ZMax gen: 163, Max flush: 278
176052024-02-20T22:48:28.975Z
176062024-02-20T22:48:28.975ZBegin loop 82
176072024-02-20T22:48:28.976ZDownstairs to restart: 1
176082024-02-20T22:48:28.976Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.475529647Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794}
176092024-02-20T22:48:28.976Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.475828496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794}
176102024-02-20T22:48:28.976Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
176112024-02-20T22:48:28.976Z,"time":"2024-02-20T22:48:28.475859405Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794}
176122024-02-20T22:48:28.976Z{"msg":"Crucible f4fe9a05-341e-4285-b42d-427e96d49ba9 has session id: c9c69c3d-16ee-4503-a6d1-62e6340a4a8f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.475903554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176132024-02-20T22:48:28.976Z{"msg":"Upstairs opts: Upstairs UUID: f4fe9a05-341e-4285-b42d-427e96d49ba9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.475954352Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176142024-02-20T22:48:28.976Z{"msg":"Crucible stats registered with UUID: f4fe9a05-341e-4285-b42d-427e96d49ba9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.475975831Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176152024-02-20T22:48:28.976Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47600107Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794}
176162024-02-20T22:48:28.976Z{"msg":"f4fe9a05-341e-4285-b42d-427e96d49ba9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476040369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176172024-02-20T22:48:28.976Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476069518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176182024-02-20T22:48:28.976Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476116156Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176192024-02-20T22:48:28.976Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476163024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176202024-02-20T22:48:28.976Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476248781Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176212024-02-20T22:48:28.976Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476374787Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176222024-02-20T22:48:28.976Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476523581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176232024-02-20T22:48:28.976Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47656562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176242024-02-20T22:48:28.976Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476612988Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176252024-02-20T22:48:28.976Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476653917Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176262024-02-20T22:48:28.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476695405Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176272024-02-20T22:48:28.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476767283Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176282024-02-20T22:48:28.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.476792762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176292024-02-20T22:48:28.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477168548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176302024-02-20T22:48:28.977Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477216927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176312024-02-20T22:48:28.977Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477243606Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176322024-02-20T22:48:28.977Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477272325Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176332024-02-20T22:48:28.977Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477298164Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176342024-02-20T22:48:28.977Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477318783Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176352024-02-20T22:48:28.977Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.477854554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176362024-02-20T22:48:28.977Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47796129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176372024-02-20T22:48:28.977Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478081306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176382024-02-20T22:48:28.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47852936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176392024-02-20T22:48:28.977Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478572788Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176402024-02-20T22:48:28.977Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478594617Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176412024-02-20T22:48:28.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478616406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176422024-02-20T22:48:28.977Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478630496Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176432024-02-20T22:48:28.977Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478645056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176442024-02-20T22:48:28.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478657445Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176452024-02-20T22:48:28.977Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478677634Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176462024-02-20T22:48:28.977Z{"msg":"[0]R flush_numbers[0..12]: [278, 278, 273, 275, 260, 278, 278, 246, 270, 271, 266, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478692014Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176472024-02-20T22:48:28.977Z{"msg":"[0]R generation[0..12]: [163, 163, 160, 161, 152, 163, 163, 141, 158, 158, 155, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478712663Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176482024-02-20T22:48:28.977Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478727073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176492024-02-20T22:48:28.977Z{"msg":"[1]R flush_numbers[0..12]: [278, 278, 273, 275, 260, 278, 278, 246, 270, 271, 266, 276]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176502024-02-20T22:48:28.977Z,"time":"2024-02-20T22:48:28.478748042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176512024-02-20T22:48:28.977Z{"msg":"[1]R generation[0..12]: [163, 163, 160, 161, 152, 163, 163, 141, 158, 158, 155, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478775591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176522024-02-20T22:48:28.977Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47879963Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176532024-02-20T22:48:28.977Z{"msg":"[2]R flush_numbers[0..12]: [278, 278, 273, 275, 260, 278, 278, 246, 270, 271, 266, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478834699Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176542024-02-20T22:48:28.977Z{"msg":"[2]R generation[0..12]: [163, 163, 160, 161, 152, 163, 163, 141, 158, 158, 155, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478852298Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176552024-02-20T22:48:28.978Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478874177Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176562024-02-20T22:48:28.978Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478888117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176572024-02-20T22:48:28.978Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478901706Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176582024-02-20T22:48:28.978Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478921455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176592024-02-20T22:48:28.978Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478935515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176602024-02-20T22:48:28.978Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478957584Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176612024-02-20T22:48:28.978Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478969934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176622024-02-20T22:48:28.978Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478984333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176632024-02-20T22:48:28.978Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.478997763Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176642024-02-20T22:48:28.978Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.479018292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176652024-02-20T22:48:28.978Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.479031212Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176662024-02-20T22:48:28.978Z{"msg":"f4fe9a05-341e-4285-b42d-427e96d49ba9 is now active with session: c9c69c3d-16ee-4503-a6d1-62e6340a4a8f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.479044451Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176672024-02-20T22:48:28.978Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47906323Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794,"session_id":"c9c69c3d-16ee-4503-a6d1-62e6340a4a8f"}
176682024-02-20T22:48:28.978Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:28.47907625Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794}
176692024-02-20T22:48:28.978ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176702024-02-20T22:48:28.978ZLoading write count information from file "/tmp/test_repair_verify.data"
176712024-02-20T22:48:28.978ZRun Repair workload
176722024-02-20T22:48:28.983Z01/30 Write block 535 len 28672 data: 18 17 15 19 15 15 16
176732024-02-20T22:48:28.983Z02/30 Read block 85 len 32768
176742024-02-20T22:48:28.983Z03/30 Read block 260 len 4096
176752024-02-20T22:48:28.983Z04/30 Read block 466 len 20480
176762024-02-20T22:48:28.983Z05/30 Read block 167 len 32768
176772024-02-20T22:48:28.983Z06/30 Write block 423 len 36864 data: 11 10 13 11 9 10 10 9 8
176782024-02-20T22:48:28.983Z07/30 Read block 333 len 4096
176792024-02-20T22:48:28.983Z08/30 Read block 206 len 32768
176802024-02-20T22:48:28.983Z09/30 Read block 275 len 28672
176812024-02-20T22:48:28.983Z10/30 Write block 487 len 12288 data: 13 13 11
176822024-02-20T22:48:28.983Z11/30 Read block 137 len 36864
176832024-02-20T22:48:28.983Z12/30 Write block 583 len 4096 data: 15
176842024-02-20T22:48:28.988Z13/30 Write block 285 len 16384 data: 13 12 13 12
176852024-02-20T22:48:28.988Z14/30 Write block 106 len 12288 data: 9 9 12
176862024-02-20T22:48:28.988Z15/30 Write block 280 len 8192 data: 9 11
176872024-02-20T22:48:28.988Z16/30 Write block 97 len 16384 data: 11 7 12 11
176882024-02-20T22:48:28.988Z17/30 Write block 64 len 16384 data: 11 13 12 16
176892024-02-20T22:48:28.988Z18/30 Write block 169 len 12288 data: 9 11 10
176902024-02-20T22:48:28.988Z19/30 Write block 402 len 12288 data: 13 15 16
176912024-02-20T22:48:28.988Z20/30 Read block 221 len 32768
176922024-02-20T22:48:28.992Z21/30 Read block 373 len 36864
176932024-02-20T22:48:28.992Z22/30 Read block 338 len 12288
176942024-02-20T22:48:28.992Z23/30 Read block 40 len 32768
176952024-02-20T22:48:28.992Z24/30 Flush
176962024-02-20T22:48:28.998Z25/30 Write block 202 len 20480 data: 7 9 10 11 13
176972024-02-20T22:48:29.004Z26/30 Write block 369 len 36864 data: 15 18 21 18 15 12 11 12 15
176982024-02-20T22:48:29.004Z27/30 Write block 433 len 4096 data: 7
176992024-02-20T22:48:29.004Z28/30 Read block 217 len 16384
177002024-02-20T22:48:29.004Z29/30 Write block 327 len 24576 data: 17 16 14 11 10 15
177012024-02-20T22:48:29.004Z30/30 Read block 149 len 4096
177022024-02-20T22:48:29.004Z----------------------------------------------------------------
177032024-02-20T22:48:29.004Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177042024-02-20T22:48:29.004ZStates: Active Active Active
177052024-02-20T22:48:29.004ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177062024-02-20T22:48:29.004Z 1 Acked 1000 Write 7 Done Done Done false
177072024-02-20T22:48:29.004Z 2 Acked 1001 Read 8 Done Done Done false
177082024-02-20T22:48:29.004Z 3 Acked 1002 Read 1 Done Done Done false
177092024-02-20T22:48:29.004Z 4 Acked 1003 Read 5 Done Done Done false
177102024-02-20T22:48:29.004Z 5 Acked 1004 Read 8 Done Done Done false
177112024-02-20T22:48:29.004Z 6 Acked 1005 Write 9 Done Done Done false
177122024-02-20T22:48:29.004Z 7 Acked 1006 Read 1 Done Done Done false
177132024-02-20T22:48:29.004Z 8 Acked 1007 Read 8 Done Sent Done false
177142024-02-20T22:48:29.004Z 9 Acked 1008 Read 7 Done Sent Done false
177152024-02-20T22:48:29.004Z 10 Acked 1009 Write 3 Done Sent Done false
177162024-02-20T22:48:29.004Z 11 Acked 1010 Read 9 Done Sent Done false
177172024-02-20T22:48:29.004Z 12 Acked 1011 Write 1 Done Sent Done false
177182024-02-20T22:48:29.004Z 13 Acked 1012 Write 4 Done Sent Done false
177192024-02-20T22:48:29.005Z 14 Acked 1013 Write 3 Done Sent Done false
177202024-02-20T22:48:29.005Z 15 Acked 1014 Write 2 Done Sent Done false
177212024-02-20T22:48:29.005Z 16 Acked 1015 Write 4 Done Sent Done false
177222024-02-20T22:48:29.005Z 17 Acked 1016 Write 4 Done Sent Done false
177232024-02-20T22:48:29.005Z 18 Acked 1017 Write 3 Done Sent Done false
177242024-02-20T22:48:29.005Z 19 Acked 1018 Write 3 Done Sent Done false
177252024-02-20T22:48:29.005Z 20 Acked 1019 Read 8 Done Sent Done false
177262024-02-20T22:48:29.005Z 21 Acked 1020 Read 9 Done Sent Done false
177272024-02-20T22:48:29.005Z 22 Acked 1021 Read 3 Done Sent Done false
177282024-02-20T22:48:29.005Z 23 Acked 1022 Read 8 Done Sent Done false
177292024-02-20T22:48:29.005Z 24 Acked 1023 Flush 0 Done Sent Done false
177302024-02-20T22:48:29.006Z 25 Acked 1024 Write 5 Done Sent Done false
177312024-02-20T22:48:29.006Z 26 Acked 1025 Write 9 Done Sent Done false
177322024-02-20T22:48:29.006Z 27 Acked 1026 Write 1 Done Sent Done false
177332024-02-20T22:48:29.006Z 28 Acked 1027 Read 4 Done Sent Done false
177342024-02-20T22:48:29.006Z 29 Acked 1028 Write 6 Done Sent Done false
177352024-02-20T22:48:29.006Z 30 Acked 1029 Read 1 Sent Sent Done false
177362024-02-20T22:48:29.006Z STATES DS:0 DS:1 DS:2 TOTAL
177372024-02-20T22:48:29.006Z New 0 0 0 0
177382024-02-20T22:48:29.006Z Sent 1 23 0 24
177392024-02-20T22:48:29.006Z Done 29 7 30 66
177402024-02-20T22:48:29.006Z Skipped 0 0 0 0
177412024-02-20T22:48:29.006Z Error 0 0 0 0
177422024-02-20T22:48:29.006ZLast Flush: 1023 0 1023
177432024-02-20T22:48:29.006ZDownstairs last five completed:
177442024-02-20T22:48:29.007ZUpstairs last five completed: 30 29 28 27 26
177452024-02-20T22:48:29.007Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:28.509066617Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1794}
177462024-02-20T22:48:29.007ZWrote out file "/tmp/test_repair_verify.data"
177472024-02-20T22:48:29.012Z
177482024-02-20T22:48:29.012ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177492024-02-20T22:48:29.023ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177502024-02-20T22:48:29.024Z 0 000-019 163 163 163 278 278 278 F F F
177512024-02-20T22:48:29.024Z 1 020-039 163 163 163 278 278 278 F F F
177522024-02-20T22:48:29.024Z 2 040-059 160 160 160 273 273 273 F F F
177532024-02-20T22:48:29.024Z 3 060-079 164 161 164 279 275 279 F F F <---
177542024-02-20T22:48:29.024Z 4 080-099 164 152 164 279 260 279 F F F <---
177552024-02-20T22:48:29.024Z 5 100-119 164 163 164 279 278 279 F F F <---
177562024-02-20T22:48:29.029Z 6 120-139 163 163 163 278 278 278 F F F
177572024-02-20T22:48:29.029Z 7 140-159 141 141 141 246 246 246 F F F
177582024-02-20T22:48:29.029Z 8 160-179 164 158 164 279 270 279 F F F <---
177592024-02-20T22:48:29.029Z 9 180-199 158 158 158 271 271 271 F F F
177602024-02-20T22:48:29.029Z 10 200-219 155 155 155 266 266 266 T F T <---
177612024-02-20T22:48:29.029Z 11 220-239 162 162 162 276 276 276 F F F
177622024-02-20T22:48:29.029Z 12 240-259 162 162 162 276 276 276 F F F
177632024-02-20T22:48:29.029Z 13 260-279 163 163 163 278 278 278 F F F
177642024-02-20T22:48:29.029Z 14 280-299 164 163 164 279 278 279 F F F <---
177652024-02-20T22:48:29.030Z 15 300-319 162 162 162 276 276 276 F F F
177662024-02-20T22:48:29.030Z 16 320-339 163 163 163 278 278 278 T F T <---
177672024-02-20T22:48:29.030Z 17 340-359 163 163 163 278 278 278 F F F
177682024-02-20T22:48:29.030Z 18 360-379 159 159 159 272 272 272 T F T <---
177692024-02-20T22:48:29.030Z 19 380-399 163 163 163 278 278 278 F F F
177702024-02-20T22:48:29.030Z 20 400-419 164 160 164 279 274 279 F F F <---
177712024-02-20T22:48:29.030Z 21 420-439 164 163 164 279 278 279 T T T <---
177722024-02-20T22:48:29.030Z 22 440-459 163 163 163 278 278 278 F F F
177732024-02-20T22:48:29.030Z 23 460-479 162 162 162 276 276 276 F F F
177742024-02-20T22:48:29.030Z 24 480-499 164 163 164 279 278 279 F F F <---
177752024-02-20T22:48:29.030Z 25 500-519 159 159 159 272 272 272 F F F
177762024-02-20T22:48:29.030Z 26 520-539 164 160 164 279 273 279 F T F <---
177772024-02-20T22:48:29.030Z 27 540-559 164 162 164 279 277 279 F T F <---
177782024-02-20T22:48:29.031Z 28 560-579 162 162 162 277 277 277 F F F
177792024-02-20T22:48:29.031Z 29 580-599 164 159 164 279 272 279 F F F <---
177802024-02-20T22:48:29.031ZMax gen: 164, Max flush: 279
177812024-02-20T22:48:29.031ZError: Difference in extent metadata found!
177822024-02-20T22:48:29.031ZA Difference in extent metadata is expected here
177832024-02-20T22:48:29.031ZOn loop 82
177842024-02-20T22:48:29.031Z
177852024-02-20T22:48:29.035ZVerifying data now
177862024-02-20T22:48:34.718ZNo change after verify
177872024-02-20T22:48:34.718ZLoop: 82 Downstairs dump after verify (and repair):
177882024-02-20T22:48:34.729ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177892024-02-20T22:48:34.729Z 0 000-019 163 163 163 278 278 278 F F F
177902024-02-20T22:48:34.729Z 1 020-039 163 163 163 278 278 278 F F F
177912024-02-20T22:48:34.729Z 2 040-059 160 160 160 273 273 273 F F F
177922024-02-20T22:48:34.729Z 3 060-079 164 164 164 279 279 279 F F F
177932024-02-20T22:48:34.729Z 4 080-099 164 164 164 279 279 279 F F F
177942024-02-20T22:48:34.729Z 5 100-119 164 164 164 279 279 279 F F F
177952024-02-20T22:48:34.729Z 6 120-139 163 163 163 278 278 278 F F F
177962024-02-20T22:48:34.730Z 7 140-159 141 141 141 246 246 246 F F F
177972024-02-20T22:48:34.730Z 8 160-179 164 164 164 279 279 279 F F F
177982024-02-20T22:48:34.730Z 9 180-199 158 158 158 271 271 271 F F F
177992024-02-20T22:48:34.730Z 10 200-219 165 165 165 280 280 280 F F F
178002024-02-20T22:48:34.743Z 11 220-239 162 162 162 276 276 276 F F F
178012024-02-20T22:48:34.743Z 12 240-259 162 162 162 276 276 276 F F F
178022024-02-20T22:48:34.744Z 13 260-279 163 163 163 278 278 278 F F F
178032024-02-20T22:48:34.744Z 14 280-299 164 164 164 279 279 279 F F F
178042024-02-20T22:48:34.744Z 15 300-319 162 162 162 276 276 276 F F F
178052024-02-20T22:48:34.744Z 16 320-339 165 165 165 280 280 280 F F F
178062024-02-20T22:48:34.744Z 17 340-359 163 163 163 278 278 278 F F F
178072024-02-20T22:48:34.744Z 18 360-379 165 165 165 280 280 280 F F F
178082024-02-20T22:48:34.745Z 19 380-399 163 163 163 278 278 278 F F F
178092024-02-20T22:48:34.745Z 20 400-419 164 164 164 279 279 279 F F F
178102024-02-20T22:48:34.745Z 21 420-439 165 165 165 280 280 280 F F F
178112024-02-20T22:48:34.745Z 22 440-459 163 163 163 278 278 278 F F F
178122024-02-20T22:48:34.745Z 23 460-479 162 162 162 276 276 276 F F F
178132024-02-20T22:48:34.745Z 24 480-499 164 164 164 279 279 279 F F F
178142024-02-20T22:48:34.745Z 25 500-519 159 159 159 272 272 272 F F F
178152024-02-20T22:48:34.745Z 26 520-539 164 164 164 279 279 279 F F F
178162024-02-20T22:48:34.745Z 27 540-559 164 164 164 279 279 279 F F F
178172024-02-20T22:48:34.746Z 28 560-579 162 162 162 277 277 277 F F F
178182024-02-20T22:48:34.746Z 29 580-599 164 164 164 279 279 279 F F F
178192024-02-20T22:48:34.746ZMax gen: 165, Max flush: 280
178202024-02-20T22:48:34.746Z
178212024-02-20T22:48:34.746ZBegin loop 83
178222024-02-20T22:48:34.746ZDownstairs to restart: 1
178232024-02-20T22:48:34.785Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.258347731Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802}
178242024-02-20T22:48:34.785Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.25868017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802}
178252024-02-20T22:48:34.785Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.258728168Z",Crucible runtime is spawned
178262024-02-20T22:48:34.785Z"hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802}
178272024-02-20T22:48:34.785Z{"msg":"Crucible 79fd5c73-cf9e-4a46-b36a-353032a8a3e2 has session id: f7686e0c-2bb1-4828-b2e3-d96d19e36a2f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.258772746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178282024-02-20T22:48:34.785Z{"msg":"Upstairs opts: Upstairs UUID: 79fd5c73-cf9e-4a46-b36a-353032a8a3e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.258838724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178292024-02-20T22:48:34.785Z{"msg":"Crucible stats registered with UUID: 79fd5c73-cf9e-4a46-b36a-353032a8a3e2","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.258890112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178302024-02-20T22:48:34.785Z{"msg":"79fd5c73-cf9e-4a46-b36a-353032a8a3e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.25895682Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178312024-02-20T22:48:34.785Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.258994388Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802}
178322024-02-20T22:48:34.785Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259009468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178332024-02-20T22:48:34.785Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259046557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178342024-02-20T22:48:34.785Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259063196Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178352024-02-20T22:48:34.785Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259092075Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178362024-02-20T22:48:34.785Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259107084Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178372024-02-20T22:48:34.785Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259127754Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178382024-02-20T22:48:34.785Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259141873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178392024-02-20T22:48:34.785Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259167992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178402024-02-20T22:48:34.786Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259182992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178412024-02-20T22:48:34.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259316967Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178422024-02-20T22:48:34.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259376955Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178432024-02-20T22:48:34.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.259435632Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178442024-02-20T22:48:34.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260134127Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178452024-02-20T22:48:34.786Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260171056Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178462024-02-20T22:48:34.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260187216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178472024-02-20T22:48:34.786Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260218555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178482024-02-20T22:48:34.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260243904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178492024-02-20T22:48:34.786Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260285452Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178502024-02-20T22:48:34.786Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260791554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178512024-02-20T22:48:34.786Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260841122Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178522024-02-20T22:48:34.786Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.260882821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178532024-02-20T22:48:34.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261414572Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178542024-02-20T22:48:34.786Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.26146822Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178552024-02-20T22:48:34.786Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261500489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178562024-02-20T22:48:34.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261542147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178572024-02-20T22:48:34.786Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261581866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178582024-02-20T22:48:34.786Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261616165Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178592024-02-20T22:48:34.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261649283Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178602024-02-20T22:48:34.786Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261684682Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178612024-02-20T22:48:34.786Z{"msg":"[0]R flush_numbers[0..12]: [278, 278, 273, 279, 279, 279, 278, 246, 279, 271, 280, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261719071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178622024-02-20T22:48:34.786Z{"msg":"[0]R generation[0..12]: [163, 163, 160, 164, 164, 164, 163, 141, 164, 158, 165, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261756129Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178632024-02-20T22:48:34.786Z{"msg":"Wait for a query_work_queue command to finish before sending IO
178642024-02-20T22:48:34.786Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261788508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178652024-02-20T22:48:34.787Z{"msg":"[1]R flush_numbers[0..12]: [278, 278, 273, 279, 279, 279, 278, 246, 279, 271, 280, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261838366Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178662024-02-20T22:48:34.787Z{"msg":"[1]R generation[0..12]: [163, 163, 160, 164, 164, 164, 163, 141, 164, 158, 165, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261878995Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178672024-02-20T22:48:34.787Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261921144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178682024-02-20T22:48:34.787Z{"msg":"[2]R flush_numbers[0..12]: [278, 278, 273, 279, 279, 279, 278, 246, 279, 271, 280, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.261961702Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178692024-02-20T22:48:34.787Z{"msg":"[2]R generation[0..12]: [163, 163, 160, 164, 164, 164, 163, 141, 164, 158, 165, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.26201871Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178702024-02-20T22:48:34.787Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262062158Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178712024-02-20T22:48:34.787Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262100247Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178722024-02-20T22:48:34.787Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262138746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178732024-02-20T22:48:34.787Z{"msg":"Next flush: 281","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178742024-02-20T22:48:34.787Z,"time":"2024-02-20T22:48:34.262175554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178752024-02-20T22:48:34.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262209223Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178762024-02-20T22:48:34.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262224883Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178772024-02-20T22:48:34.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262244332Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178782024-02-20T22:48:34.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262258162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178792024-02-20T22:48:34.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262271401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178802024-02-20T22:48:34.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262285341Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178812024-02-20T22:48:34.787Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.26229793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178822024-02-20T22:48:34.787Z{"msg":"79fd5c73-cf9e-4a46-b36a-353032a8a3e2 is now active with session: f7686e0c-2bb1-4828-b2e3-d96d19e36a2f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.26231826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178832024-02-20T22:48:34.787Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262330779Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802,"session_id":"f7686e0c-2bb1-4828-b2e3-d96d19e36a2f"}
178842024-02-20T22:48:34.787Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:34.262350298Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802}
178852024-02-20T22:48:34.787ZLoading write count information from file "/tmp/test_repair_verify.data"
178862024-02-20T22:48:34.787ZRun Repair workload
178872024-02-20T22:48:34.787Z01/30 Write block 545 len 36864 data: 12 12 13 11 11 8 7 6 8
178882024-02-20T22:48:34.799Z02/30 Read block 593 len 4096
178892024-02-20T22:48:34.799Z03/30 Read block 231 len 16384
178902024-02-20T22:48:34.799Z04/30 Write block 538 len 24576 data: 20 16 16 17 13 14
178912024-02-20T22:48:34.799Z05/30 Write block 10 len 4096 data: 12
178922024-02-20T22:48:34.799Z06/30 Read block 525 len 28672
178932024-02-20T22:48:34.799Z07/30 Read block 180 len 32768
178942024-02-20T22:48:34.799Z08/30 Read block 175 len 16384
178952024-02-20T22:48:34.799Z09/30 Write block 39 len 12288 data: 16 12 11
178962024-02-20T22:48:34.799Z10/30 Read block 492 len 4096
178972024-02-20T22:48:34.799Z11/30 Read block 160 len 4096
178982024-02-20T22:48:34.799Z12/30 Write block 558 len 24576 data: 11 8 7 10 7 8
178992024-02-20T22:48:34.799Z13/30 Flush
179002024-02-20T22:48:34.799Z14/30 Write block 118 len 24576 data: 10 14 14 16 18 16
179012024-02-20T22:48:34.799Z15/30 Read block 532 len 28672
179022024-02-20T22:48:34.799Z16/30 Read block 557 len 8192
179032024-02-20T22:48:34.799Z17/30 Write block 207 len 24576 data: 13 12 13 16 15 14
179042024-02-20T22:48:34.799Z18/30 Flush
179052024-02-20T22:48:34.799Z19/30 Write block 442 len 16384 data: 14 11 10 13
179062024-02-20T22:48:34.799Z20/30 Read block 549 len 4096
179072024-02-20T22:48:34.800Z21/30 Read block 480 len 8192
179082024-02-20T22:48:34.800Z22/30 Read block 311 len 8192
179092024-02-20T22:48:34.800Z23/30 Read block 219 len 24576
179102024-02-20T22:48:34.800Z24/30 Read block 350 len 4096
179112024-02-20T22:48:34.800Z25/30 Write block 457 len 20480 data: 19 22 20 18 21
179122024-02-20T22:48:34.800Z26/30 Flush
179132024-02-20T22:48:34.800Z27/30 Write block 26 len 24576 data: 14 19 19 24 25 25
179142024-02-20T22:48:34.800Z28/30 Write block 486 len 8192 data: 13 14
179152024-02-20T22:48:34.800Z29/30 Read block 109 len 28672
179162024-02-20T22:48:34.800Z30/30 Read block 540 len 4096
179172024-02-20T22:48:34.800Z----------------------------------------------------------------
179182024-02-20T22:48:34.800Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
179192024-02-20T22:48:34.800ZStates: Active Active Active
179202024-02-20T22:48:34.800ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179212024-02-20T22:48:34.800Z 1 Acked 1000 Write 9 Done Done Done false
179222024-02-20T22:48:34.800Z 2 Acked 1001 Read 1 Done Done Done false
179232024-02-20T22:48:34.800Z 3 Acked 1002 Read 4 Done Sent Done false
179242024-02-20T22:48:34.800Z 4 Acked 1003 Write 6 Done Sent Done false
179252024-02-20T22:48:34.800Z 5 Acked 1004 Write 1 Done Sent Done false
179262024-02-20T22:48:34.800Z 6 Acked 1005 Read 7 Done Sent Done false
179272024-02-20T22:48:34.800Z 7 Acked 1006 Read 8 Done Sent Done false
179282024-02-20T22:48:34.800Z 8 Acked 1007 Read 4 Done Sent Done false
179292024-02-20T22:48:34.800Z 9 Acked 1008 Write 3 Done Sent Done false
179302024-02-20T22:48:34.800Z 10 Acked 1009 Read 1 Done Sent Done false
179312024-02-20T22:48:34.800Z 11 Acked 1010 Read 1 Done Sent Done false
179322024-02-20T22:48:34.800Z 12 Acked 1011 Write 6 Done Sent Done false
179332024-02-20T22:48:34.801Z 13 Acked 1012 Flush 0 Done Sent Done false
179342024-02-20T22:48:34.801Z 14 Acked 1013 Write 6 Done Sent Done false
179352024-02-20T22:48:34.801Z 15 Acked 1014 Read 7 Done Sent Done false
179362024-02-20T22:48:34.801Z 16 Acked 1015 Read 2 Done Sent Done false
179372024-02-20T22:48:34.801Z 17 Acked 1016 Write 6 Done Sent Done false
179382024-02-20T22:48:34.801Z 18 Acked 1017 Flush 0 Done Sent Done false
179392024-02-20T22:48:34.801Z 19 Acked 1018 Write 4 Done Sent Done false
179402024-02-20T22:48:34.801Z 20 Acked 1019 Read 1 Done Sent Done false
179412024-02-20T22:48:34.801Z 21 Acked 1020 Read 2 Done Sent Done false
179422024-02-20T22:48:34.801Z 22 Acked 1021 Read 2 Done Sent Done false
179432024-02-20T22:48:34.801Z 23 Acked 1022 Read 6 Done Sent Done false
179442024-02-20T22:48:34.801Z 24 Acked 1023 Read 1 Done Sent Done false
179452024-02-20T22:48:34.801Z 25 Acked 1024 Write 5 Done Sent Done false
179462024-02-20T22:48:34.801Z 26 Acked 1025 Flush 0 Done Sent Done false
179472024-02-20T22:48:34.801Z 27 Acked 1026 Write 6 Done Sent Done false
179482024-02-20T22:48:34.801Z 28 Acked 1027 Write 2 Done Sent Done false
179492024-02-20T22:48:34.801Z 29 Acked 1028 Read 7 Done Sent Done false
179502024-02-20T22:48:34.801Z 30 Acked 1029 Read 1 Done Sent Done false
179512024-02-20T22:48:34.801Z STATES DS:0 DS:1 DS:2 TOTAL
179522024-02-20T22:48:34.801Z New 0 0 0 0
179532024-02-20T22:48:34.801Z Sent 0 28 0 28
179542024-02-20T22:48:34.801Z Done 30 2 30 62
179552024-02-20T22:48:34.801Z Skipped 0 0 0 0
179562024-02-20T22:48:34.801Z Error 0 0 0 0
179572024-02-20T22:48:34.801ZLast Flush: 1025 0 1025
179582024-02-20T22:48:34.801ZDownstairs last five completed:
179592024-02-20T22:48:34.802ZUpstairs last five completed: 30 29 28 27 26
179602024-02-20T22:48:34.802Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:34.291545024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1802}
179612024-02-20T22:48:34.802ZWrote out file "/tmp/test_repair_verify.data"
179622024-02-20T22:48:34.802Z
179632024-02-20T22:48:34.802ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179642024-02-20T22:48:34.808ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179652024-02-20T22:48:34.808Z 0 000-019 166 163 166 281 278 281 F F F <---
179662024-02-20T22:48:34.808Z 1 020-039 166 163 166 281 278 281 T F T <---
179672024-02-20T22:48:34.808Z 2 040-059 166 160 166 281 273 281 F F F <---
179682024-02-20T22:48:34.809Z 3 060-079 164 164 164 279 279 279 F F F
179692024-02-20T22:48:34.809Z 4 080-099 164 164 164 279 279 279 F F F
179702024-02-20T22:48:34.809Z 5 100-119 166 164 166 282 279 282 F F F <---
179712024-02-20T22:48:34.809Z 6 120-139 166 163 166 282 278 282 F F F <---
179722024-02-20T22:48:34.809Z 7 140-159 141 141 141 246 246 246 F F F
179732024-02-20T22:48:34.809Z 8 160-179 164 164 164 279 279 279 F F F
179742024-02-20T22:48:34.809Z 9 180-199 158 158 158 271 271 271 F F F
179752024-02-20T22:48:34.809Z 10 200-219 166 165 166 282 280 282 F F F <---
179762024-02-20T22:48:34.809Z 11 220-239 162 162 162 276 276 276 F F F
179772024-02-20T22:48:34.809Z 12 240-259 162 162 162 276 276 276 F F F
179782024-02-20T22:48:34.809Z 13 260-279 163 163 163 278 278 278 F F F
179792024-02-20T22:48:34.809Z 14 280-299 164 164 164 279 279 279 F F F
179802024-02-20T22:48:34.809Z 15 300-319 162 162 162 276 276 276 F F F
179812024-02-20T22:48:34.809Z 16 320-339 165 165 165 280 280 280 F F F
179822024-02-20T22:48:34.809Z 17 340-359 163 163 163 278 278 278 F F F
179832024-02-20T22:48:34.809Z 18 360-379 165 165 165 280 280 280 F F F
179842024-02-20T22:48:34.809Z 19 380-399 163 163 163 278 278 278 F F F
179852024-02-20T22:48:34.809Z 20 400-419 164 164 164 279 279 279 F F F
179862024-02-20T22:48:34.809Z 21 420-439 165 165 165 280 280 280 F F F
179872024-02-20T22:48:34.809Z 22 440-459 166 163 166 283 278 283 F F F <---
179882024-02-20T22:48:34.809Z 23 460-479 166 162 166 283 276 283 F F F <---
179892024-02-20T22:48:34.809Z 24 480-499 164 164 164 279 279 279 T F T <---
179902024-02-20T22:48:34.809Z 25 500-519 159 159 159 272 272 272 F F F
179912024-02-20T22:48:34.809Z 26 520-539 166 164 166 281 279 281 F F F <---
179922024-02-20T22:48:34.810Z 27 540-559 166 164 166 281 279 281 F T F <---
179932024-02-20T22:48:34.810Z 28 560-579 166 162 166 281 277 281 F F F <---
179942024-02-20T22:48:34.810Z 29 580-599 164 164 164 279 279 279 F F F
179952024-02-20T22:48:34.810ZMax gen: 166, Max flush: 283
179962024-02-20T22:48:34.810ZError: Difference in extent metadata found!
179972024-02-20T22:48:34.814ZA Difference in extent metadata is expected here
179982024-02-20T22:48:34.814ZOn loop 83
179992024-02-20T22:48:34.814Z
180002024-02-20T22:48:34.814ZVerifying data now
180012024-02-20T22:48:39.408ZNo change after verify
180022024-02-20T22:48:39.408ZLoop: 83 Downstairs dump after verify (and repair):
180032024-02-20T22:48:39.422ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180042024-02-20T22:48:39.422Z 0 000-019 166 166 166 281 281 281 F F F
180052024-02-20T22:48:39.422Z 1 020-039 167 167 167 284 284 284 F F F
180062024-02-20T22:48:39.422Z 2 040-059 166 166 166 281 281 281 F F F
180072024-02-20T22:48:39.422Z 3 060-079 164 164 164 279 279 279 F F F
180082024-02-20T22:48:39.422Z 4 080-099 164 164 164 279 279 279 F F F
180092024-02-20T22:48:39.422Z 5 100-119 166 166 166 282 282 282 F F F
180102024-02-20T22:48:39.422Z 6 120-139 166 166 166 282 282 282 F F F
180112024-02-20T22:48:39.422Z 7 140-159 141 141 141 246 246 246 F F F
180122024-02-20T22:48:39.422Z 8 160-179 164 164 164 279 279 279 F F F
180132024-02-20T22:48:39.422Z 9 180-199 158 158 158 271 271 271 F F F
180142024-02-20T22:48:39.423Z 10 200-219 166 166 166 282 282 282 F F F
180152024-02-20T22:48:39.423Z 11 220-239 162 162 162 276 276 276 F F F
180162024-02-20T22:48:39.423Z 12 240-259 162 162 162 276 276 276 F F F
180172024-02-20T22:48:39.423Z 13 260-279 163 163 163 278 278 278 F F F
180182024-02-20T22:48:39.428Z 14 280-299 164 164 164 279 279 279 F F F
180192024-02-20T22:48:39.428Z 15 300-319 162 162 162 276 276 276 F F F
180202024-02-20T22:48:39.428Z 16 320-339 165 165 165 280 280 280 F F F
180212024-02-20T22:48:39.428Z 17 340-359 163 163 163 278 278 278 F F F
180222024-02-20T22:48:39.428Z 18 360-379 165 165 165 280 280 280 F F F
180232024-02-20T22:48:39.428Z 19 380-399 163 163 163 278 278 278 F F F
180242024-02-20T22:48:39.428Z 20 400-419 164 164 164 279 279 279 F F F
180252024-02-20T22:48:39.428Z 21 420-439 165 165 165 280 280 280 F F F
180262024-02-20T22:48:39.428Z 22 440-459 166 166 166 283 283 283 F F F
180272024-02-20T22:48:39.428Z 23 460-479 166 166 166 283 283 283 F F F
180282024-02-20T22:48:39.428Z 24 480-499 167 167 167 284 284 284 F F F
180292024-02-20T22:48:39.428Z 25 500-519 159 159 159 272 272 272 F F F
180302024-02-20T22:48:39.428Z 26 520-539 166 166 166 281 281 281 F F F
180312024-02-20T22:48:39.429Z 27 540-559 166 166 166 281 281 281 F F F
180322024-02-20T22:48:39.429Z 28 560-579 166 166 166 281 281 281 F F F
180332024-02-20T22:48:39.429Z 29 580-599 164 164 164 279 279 279 F F F
180342024-02-20T22:48:39.429ZMax gen: 167, Max flush: 284
180352024-02-20T22:48:39.429Z
180362024-02-20T22:48:39.429ZBegin loop 84
180372024-02-20T22:48:39.429ZDownstairs to restart: 2
180382024-02-20T22:48:39.444Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.95056729Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810}
180392024-02-20T22:48:39.444Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.95086849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810}
180402024-02-20T22:48:39.444Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.950888889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810}
180412024-02-20T22:48:39.453Z{"msg":"Crucible c98cf15e-a6e0-41c6-96ea-d8a12084f1aa has session id: 25bd931b-5f10-4102-8163-fecd42f0f117","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.950907108Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180422024-02-20T22:48:39.453Z{"msg":"Upstairs opts: Upstairs UUID: c98cf15e-a6e0-41c6-96ea-d8a12084f1aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.950944767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180432024-02-20T22:48:39.453Z{"msg":"Crucible stats registered with UUID: c98cf15e-a6e0-41c6-96ea-d8a12084f1aa","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.950975996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180442024-02-20T22:48:39.453ZCrucible runtime is spawned
180452024-02-20T22:48:39.453Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951151069Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810}
180462024-02-20T22:48:39.453Z{"msg":"c98cf15e-a6e0-41c6-96ea-d8a12084f1aa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951334843Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180472024-02-20T22:48:39.453Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951370932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180482024-02-20T22:48:39.453Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951441609Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180492024-02-20T22:48:39.453Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951485097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180502024-02-20T22:48:39.453Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951514747Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180512024-02-20T22:48:39.454Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951544735Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180522024-02-20T22:48:39.454Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951572904Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180532024-02-20T22:48:39.454Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951601053Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180542024-02-20T22:48:39.454Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951644042Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180552024-02-20T22:48:39.454Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951676371Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180562024-02-20T22:48:39.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951710289Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180572024-02-20T22:48:39.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951752988Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180582024-02-20T22:48:39.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.951870784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180592024-02-20T22:48:39.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.952567979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180602024-02-20T22:48:39.454Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.952605537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180612024-02-20T22:48:39.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.952621517Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180622024-02-20T22:48:39.454Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.952636116Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180632024-02-20T22:48:39.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.952657375Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180642024-02-20T22:48:39.455Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.952671145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180652024-02-20T22:48:39.455Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953172117Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180662024-02-20T22:48:39.455Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953217115Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180672024-02-20T22:48:39.455Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953261854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180682024-02-20T22:48:39.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953695598Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180692024-02-20T22:48:39.455Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953758766Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180702024-02-20T22:48:39.455Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953794875Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180712024-02-20T22:48:39.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953830853Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180722024-02-20T22:48:39.455Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953860012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180732024-02-20T22:48:39.455Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953881112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180742024-02-20T22:48:39.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953906301Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180752024-02-20T22:48:39.455Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.95392442Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180762024-02-20T22:48:39.456Z{"msg":"[0]R flush_numbers[0..12]: [281, 284, 281, 279, 279, 282, 282, 246, 279, 271, 282, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953948679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180772024-02-20T22:48:39.456Z{"msg":"[0]R generation[0..12]: [166, 167, 166, 164, 164, 166, 166, 141, 164, 158, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953967779Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180782024-02-20T22:48:39.456Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.953991738Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180792024-02-20T22:48:39.456Z{"msg":"Wait for a query_work_queue command to finish before sending IO
180802024-02-20T22:48:39.456Z[1]R flush_numbers[0..12]: [281, 284, 281, 279, 279, 282, 282, 246, 279, 271, 282, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954030636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180812024-02-20T22:48:39.456Z{"msg":"[1]R generation[0..12]: [166, 167, 166, 164, 164, 166, 166, 141, 164, 158, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954054525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180822024-02-20T22:48:39.456Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954069295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180832024-02-20T22:48:39.456Z{"msg":"[2]R flush_numbers[0..12]: [281, 284, 281, 279, 279, 282, 282, 246, 279, 271, 282, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954102714Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180842024-02-20T22:48:39.456Z{"msg":"[2]R generation[0..12]: [166, 167, 166, 164, 164, 166, 166, 141, 164, 158, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954133893Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180852024-02-20T22:48:39.456Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954149502Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180862024-02-20T22:48:39.456Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954169321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180872024-02-20T22:48:39.456Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954186841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180882024-02-20T22:48:39.456Z{"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.95421816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180892024-02-20T22:48:39.457Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954234049Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180902024-02-20T22:48:39.457Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954254318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180912024-02-20T22:48:39.457Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954268718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180922024-02-20T22:48:39.457Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954300537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180932024-02-20T22:48:39.457Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954317116Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180942024-02-20T22:48:39.457Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954343235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180952024-02-20T22:48:39.457Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954358465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180962024-02-20T22:48:39.457Z{"msg":"c98cf15e-a6e0-41c6-96ea-d8a12084f1aa is now active with session: 25bd931b-5f10-4102-8163-fecd42f0f117","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954389473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180972024-02-20T22:48:39.457Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954405633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810,"session_id":"25bd931b-5f10-4102-8163-fecd42f0f117"}
180982024-02-20T22:48:39.457Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:38.954430562Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810}
180992024-02-20T22:48:39.457ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181002024-02-20T22:48:39.457ZLoading write count information from file "/tmp/test_repair_verify.data"
181012024-02-20T22:48:39.457ZRun Repair workload
181022024-02-20T22:48:39.458Z01/30 Write block 456 len 20480 data: 19 20 23 21 19
181032024-02-20T22:48:39.463Z02/30 Read block 36 len 12288
181042024-02-20T22:48:39.463Z03/30 Flush
181052024-02-20T22:48:39.463Z04/30 Write block 438 len 12288 data: 9 12 15
181062024-02-20T22:48:39.463Z05/30 Read block 529 len 32768
181072024-02-20T22:48:39.463Z06/30 Write block 454 len 20480 data: 12 13 20 21 24
181082024-02-20T22:48:39.463Z07/30 Write block 407 len 16384 data: 13 11 13 14
181092024-02-20T22:48:39.463Z08/30 Write block 113 len 20480 data: 13 12 10 13 14
181102024-02-20T22:48:39.463Z09/30 Write block 72 len 4096 data: 15
181112024-02-20T22:48:39.463Z10/30 Write block 589 len 20480 data: 10 12 14 12 12
181122024-02-20T22:48:39.463Z11/30 Flush
181132024-02-20T22:48:39.464Z12/30 Write block 538 len 8192 data: 21 17
181142024-02-20T22:48:39.464Z13/30 Read block 254 len 20480
181152024-02-20T22:48:39.464Z14/30 Write block 249 len 8192 data: 9 11
181162024-02-20T22:48:39.464Z15/30 Flush
181172024-02-20T22:48:39.468Z16/30 Write block 377 len 16384 data: 16 13 12 12
181182024-02-20T22:48:39.468Z17/30 Read block 424 len 8192
181192024-02-20T22:48:39.468Z18/30 Write block 212 len 24576 data: 15 9 9 10 9 6
181202024-02-20T22:48:39.469Z19/30 Read block 105 len 28672
181212024-02-20T22:48:39.469Z20/30 Write block 475 len 32768 data: 12 14 19 21 17 20 19 19
181222024-02-20T22:48:39.469Z21/30 Read block 152 len 8192
181232024-02-20T22:48:39.476Z22/30 Flush
181242024-02-20T22:48:39.476Z23/30 Write block 140 len 12288 data: 11 11 13
181252024-02-20T22:48:39.476Z24/30 Read block 502 len 12288
181262024-02-20T22:48:39.477Z25/30 Flush
181272024-02-20T22:48:39.477Z26/30 Flush
181282024-02-20T22:48:39.485Z27/30 Write block 46 len 40960 data: 8 9 12 13 11 11 12 10 13 11
181292024-02-20T22:48:39.485Z28/30 Read block 284 len 36864
181302024-02-20T22:48:39.485Z29/30 Read block 13 len 8192
181312024-02-20T22:48:39.485Z30/30 Read block 410 len 24576
181322024-02-20T22:48:39.485Z----------------------------------------------------------------
181332024-02-20T22:48:39.486Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:27
181342024-02-20T22:48:39.486ZStates: Active Active Active
181352024-02-20T22:48:39.486ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181362024-02-20T22:48:39.486Z 4 Acked 1003 Write 3 Done Done Done false
181372024-02-20T22:48:39.486Z 5 Acked 1004 Read 8 Done Done Done false
181382024-02-20T22:48:39.486Z 6 Acked 1005 Write 5 Done Done Done false
181392024-02-20T22:48:39.486Z 7 Acked 1006 Write 4 Done Done Sent false
181402024-02-20T22:48:39.486Z 8 Acked 1007 Write 5 Done Done Sent false
181412024-02-20T22:48:39.486Z 9 Acked 1008 Write 1 Done Done Sent false
181422024-02-20T22:48:39.486Z 10 Acked 1009 Write 5 Done Done Sent false
181432024-02-20T22:48:39.486Z 11 Acked 1010 Flush 0 Done Done Sent false
181442024-02-20T22:48:39.486Z 12 Acked 1011 Write 2 Done Done Sent false
181452024-02-20T22:48:39.486Z 13 Acked 1012 Read 5 Done Done Sent false
181462024-02-20T22:48:39.486Z 14 Acked 1013 Write 2 Done Done Sent false
181472024-02-20T22:48:39.486Z 15 Acked 1014 Flush 0 Done Done Sent false
181482024-02-20T22:48:39.486Z 16 Acked 1015 Write 4 Done Done Sent false
181492024-02-20T22:48:39.486Z 17 Acked 1016 Read 2 Done Done Sent false
181502024-02-20T22:48:39.486Z 18 Acked 1017 Write 6 Done Done Sent false
181512024-02-20T22:48:39.486Z 19 Acked 1018 Read 7 Done Done Sent false
181522024-02-20T22:48:39.486Z 20 Acked 1019 Write 8 Done Done Sent false
181532024-02-20T22:48:39.486Z 21 Acked 1020 Read 2 Done Done Sent false
181542024-02-20T22:48:39.486Z 22 Acked 1021 Flush 0 Done Done Sent false
181552024-02-20T22:48:39.487Z 23 Acked 1022 Write 3 Done Done Sent false
181562024-02-20T22:48:39.487Z 24 Acked 1023 Read 3 Done Done Sent false
181572024-02-20T22:48:39.487Z 25 Acked 1024 Flush 0 Done Done Sent false
181582024-02-20T22:48:39.487Z 26 Acked 1025 Flush 0 Done Done Sent false
181592024-02-20T22:48:39.487Z 27 Acked 1026 Write 10 Done Done Sent false
181602024-02-20T22:48:39.487Z 28 Acked 1027 Read 9 Done Done Sent false
181612024-02-20T22:48:39.487Z 29 Acked 1028 Read 2 Done Done Sent false
181622024-02-20T22:48:39.487Z 30 Acked 1029 Read 6 Done Done Sent false
181632024-02-20T22:48:39.487Z STATES DS:0 DS:1 DS:2 TOTAL
181642024-02-20T22:48:39.487Z New 0 0 0 0
181652024-02-20T22:48:39.487Z Sent 0 0 24 24
181662024-02-20T22:48:39.487Z Done 27 27 3 57
181672024-02-20T22:48:39.487Z Skipped 0 0 0 0
181682024-02-20T22:48:39.487Z Error 0 0 0 0
181692024-02-20T22:48:39.487ZLast Flush: 1025 1025 1002
181702024-02-20T22:48:39.487ZDownstairs last five completed: 1002 1001 1000
181712024-02-20T22:48:39.487ZUpstairs last five completed: 30 29 28 27 26
181722024-02-20T22:48:39.487Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:38.98857151Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1810}
181732024-02-20T22:48:39.487ZWrote out file "/tmp/test_repair_verify.data"
181742024-02-20T22:48:39.492Z
181752024-02-20T22:48:39.492ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181762024-02-20T22:48:39.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181772024-02-20T22:48:39.504Z 0 000-019 166 166 166 281 281 281 F F F
181782024-02-20T22:48:39.504Z 1 020-039 167 167 167 284 284 284 F F F
181792024-02-20T22:48:39.505Z 2 040-059 166 166 166 281 281 281 T T F <---
181802024-02-20T22:48:39.505Z 3 060-079 168 168 164 286 286 279 F F F <---
181812024-02-20T22:48:39.505Z 4 080-099 164 164 164 279 279 279 F F F
181822024-02-20T22:48:39.505Z 5 100-119 168 168 166 286 286 282 F F F <---
181832024-02-20T22:48:39.505Z 6 120-139 166 166 166 282 282 282 F F F
181842024-02-20T22:48:39.505Z 7 140-159 168 168 141 289 289 246 F F F <---
181852024-02-20T22:48:39.510Z 8 160-179 164 164 164 279 279 279 F F F
181862024-02-20T22:48:39.510Z 9 180-199 158 158 158 271 271 271 F F F
181872024-02-20T22:48:39.510Z 10 200-219 168 168 166 288 288 282 F F F <---
181882024-02-20T22:48:39.510Z 11 220-239 162 162 162 276 276 276 F F F
181892024-02-20T22:48:39.510Z 12 240-259 168 168 162 287 287 276 F F F <---
181902024-02-20T22:48:39.510Z 13 260-279 163 163 163 278 278 278 F F F
181912024-02-20T22:48:39.511Z 14 280-299 164 164 164 279 279 279 F F F
181922024-02-20T22:48:39.511Z 15 300-319 162 162 162 276 276 276 F F F
181932024-02-20T22:48:39.511Z 16 320-339 165 165 165 280 280 280 F F F
181942024-02-20T22:48:39.511Z 17 340-359 163 163 163 278 278 278 F F F
181952024-02-20T22:48:39.511Z 18 360-379 168 168 165 288 288 280 F F F <---
181962024-02-20T22:48:39.511Z 19 380-399 168 168 163 288 288 278 F F F <---
181972024-02-20T22:48:39.511Z 20 400-419 168 168 164 286 286 279 F F F <---
181982024-02-20T22:48:39.511Z 21 420-439 168 168 165 286 286 280 F F T <---
181992024-02-20T22:48:39.511Z 22 440-459 168 168 168 286 286 285 F F T <---
182002024-02-20T22:48:39.511Z 23 460-479 168 168 168 288 288 285 F F F <---
182012024-02-20T22:48:39.511Z 24 480-499 168 168 167 288 288 284 F F F <---
182022024-02-20T22:48:39.511Z 25 500-519 159 159 159 272 272 272 F F F
182032024-02-20T22:48:39.511Z 26 520-539 168 168 166 287 287 281 F F F <---
182042024-02-20T22:48:39.512Z 27 540-559 166 166 166 281 281 281 F F F
182052024-02-20T22:48:39.512Z 28 560-579 166 166 166 281 281 281 F F F
182062024-02-20T22:48:39.512Z 29 580-599 168 168 164 286 286 279 F F F <---
182072024-02-20T22:48:39.512ZMax gen: 168, Max flush: 289
182082024-02-20T22:48:39.512ZError: Difference in extent metadata found!
182092024-02-20T22:48:39.512ZA Difference in extent metadata is expected here
182102024-02-20T22:48:39.512ZOn loop 84
182112024-02-20T22:48:39.512Z
182122024-02-20T22:48:39.516ZVerifying data now
182132024-02-20T22:48:44.228ZNo change after verify
182142024-02-20T22:48:44.228ZLoop: 84 Downstairs dump after verify (and repair):
182152024-02-20T22:48:44.242ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182162024-02-20T22:48:44.242Z 0 000-019 166 166 166 281 281 281 F F F
182172024-02-20T22:48:44.242Z 1 020-039 167 167 167 284 284 284 F F F
182182024-02-20T22:48:44.242Z 2 040-059 169 169 169 290 290 290 F F F
182192024-02-20T22:48:44.242Z 3 060-079 168 168 168 286 286 286 F F F
182202024-02-20T22:48:44.242Z 4 080-099 164 164 164 279 279 279 F F F
182212024-02-20T22:48:44.243Z 5 100-119 168 168 168 286 286 286 F F F
182222024-02-20T22:48:44.243Z 6 120-139 166 166 166 282 282 282 F F F
182232024-02-20T22:48:44.243Z 7 140-159 168 168 168 289 289 289 F F F
182242024-02-20T22:48:44.247Z 8 160-179 164 164 164 279 279 279 F F F
182252024-02-20T22:48:44.247Z 9 180-199 158 158 158 271 271 271 F F F
182262024-02-20T22:48:44.247Z 10 200-219 168 168 168 288 288 288 F F F
182272024-02-20T22:48:44.247Z 11 220-239 162 162 162 276 276 276 F F F
182282024-02-20T22:48:44.247Z 12 240-259 168 168 168 287 287 287 F F F
182292024-02-20T22:48:44.247Z 13 260-279 163 163 163 278 278 278 F F F
182302024-02-20T22:48:44.248Z 14 280-299 164 164 164 279 279 279 F F F
182312024-02-20T22:48:44.248Z 15 300-319 162 162 162 276 276 276 F F F
182322024-02-20T22:48:44.248Z 16 320-339 165 165 165 280 280 280 F F F
182332024-02-20T22:48:44.248Z 17 340-359 163 163 163 278 278 278 F F F
182342024-02-20T22:48:44.248Z 18 360-379 168 168 168 288 288 288 F F F
182352024-02-20T22:48:44.248Z 19 380-399 168 168 168 288 288 288 F F F
182362024-02-20T22:48:44.248Z 20 400-419 168 168 168 286 286 286 F F F
182372024-02-20T22:48:44.248Z 21 420-439 168 168 168 286 286 286 F F F
182382024-02-20T22:48:44.248Z 22 440-459 168 168 168 286 286 286 F F F
182392024-02-20T22:48:44.248Z 23 460-479 168 168 168 288 288 288 F F F
182402024-02-20T22:48:44.248Z 24 480-499 168 168 168 288 288 288 F F F
182412024-02-20T22:48:44.248Z 25 500-519 159 159 159 272 272 272 F F F
182422024-02-20T22:48:44.248Z 26 520-539 168 168 168 287 287 287 F F F
182432024-02-20T22:48:44.248Z 27 540-559 166 166 166 281 281 281 F F F
182442024-02-20T22:48:44.248Z 28 560-579 166 166 166 281 281 281 F F F
182452024-02-20T22:48:44.248Z 29 580-599 168 168 168 286 286 286 F F F
182462024-02-20T22:48:44.248ZMax gen: 169, Max flush: 290
182472024-02-20T22:48:44.248Z
182482024-02-20T22:48:44.248ZBegin loop 85
182492024-02-20T22:48:44.248ZDownstairs to restart: 1
182502024-02-20T22:48:44.264Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.771424276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818}
182512024-02-20T22:48:44.265Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.771705566Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818}
182522024-02-20T22:48:44.273Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.771744145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818}
182532024-02-20T22:48:44.273Z{"msg":"Crucible 7c172915-8586-4b16-9916-c8a0b8cb5e10 has session id: 76a44ea2-ddf3-439f-877e-d2d14d320ed5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.771807012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182542024-02-20T22:48:44.273Z{"msg":"Upstairs opts: Upstairs UUID: 7c172915-8586-4b16-9916-c8a0b8cb5e10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
182552024-02-20T22:48:44.273Z,"time":"2024-02-20T22:48:43.771858431Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182562024-02-20T22:48:44.274Z{"msg":"Crucible stats registered with UUID: 7c172915-8586-4b16-9916-c8a0b8cb5e10","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.7718765Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182572024-02-20T22:48:44.274Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.771963147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818}
182582024-02-20T22:48:44.274Z{"msg":"7c172915-8586-4b16-9916-c8a0b8cb5e10 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.771989136Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182592024-02-20T22:48:44.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772002816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182602024-02-20T22:48:44.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772029255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182612024-02-20T22:48:44.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772047184Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182622024-02-20T22:48:44.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772067593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182632024-02-20T22:48:44.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772085023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182642024-02-20T22:48:44.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772128811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182652024-02-20T22:48:44.274Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.77216587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182662024-02-20T22:48:44.274Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772214418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182672024-02-20T22:48:44.274Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772241677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182682024-02-20T22:48:44.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772327204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182692024-02-20T22:48:44.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772380992Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182702024-02-20T22:48:44.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.772412141Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182712024-02-20T22:48:44.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773072267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182722024-02-20T22:48:44.274Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773109776Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182732024-02-20T22:48:44.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773129785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182742024-02-20T22:48:44.275Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773151995Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182752024-02-20T22:48:44.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773166204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182762024-02-20T22:48:44.275Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773197263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182772024-02-20T22:48:44.275Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773743093Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182782024-02-20T22:48:44.275Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773798381Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182792024-02-20T22:48:44.275Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.773922297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182802024-02-20T22:48:44.275Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774269195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182812024-02-20T22:48:44.275Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774321263Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182822024-02-20T22:48:44.275Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774351041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182832024-02-20T22:48:44.275Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774371361Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182842024-02-20T22:48:44.275Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.77440089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182852024-02-20T22:48:44.275Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774415409Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182862024-02-20T22:48:44.275Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774430299Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182872024-02-20T22:48:44.275Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774457318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182882024-02-20T22:48:44.275Z{"msg":"[0]R flush_numbers[0..12]: [281, 284, 290, 286, 279, 286, 282, 289, 279, 271, 288, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774472637Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182892024-02-20T22:48:44.275Z{"msg":"[0]R generation[0..12]: [166, 167, 169, 168, 164, 168, 166, 168, 164, 158, 168, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774486517Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182902024-02-20T22:48:44.275Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774507756Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182912024-02-20T22:48:44.275Z{"msg":"[1]R flush_numbers[0..12]: [281, 284, 290, 286, 279, 286, 282, 289, 279, 271, 288, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774522295Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182922024-02-20T22:48:44.275Z{"msg":"[1]R generation[0..12]: [166, 167, 169, 168, 164, 168, 166, 168, 164, 158, 168, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774536785Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182932024-02-20T22:48:44.275Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774550874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182942024-02-20T22:48:44.275Z{"msg":"[2]R flush_numbers[0..12]: [281, 284, 290, 286, 279, 286, 282, 289, 279, 271, 288, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774565574Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182952024-02-20T22:48:44.275Z{"msg":"[2]R generation[0..12]: [166, 167, 169, 168, 164, 168, 166, 168, 164, 158, 168, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774579243Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182962024-02-20T22:48:44.276Z{"msg":"Wait for a query_work_queue command to finish before sending IO
182972024-02-20T22:48:44.276Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774603203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182982024-02-20T22:48:44.276Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774617422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
182992024-02-20T22:48:44.276Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774631382Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183002024-02-20T22:48:44.276Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774645071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183012024-02-20T22:48:44.276Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774658321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183022024-02-20T22:48:44.276Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183032024-02-20T22:48:44.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.77468455Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183042024-02-20T22:48:44.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774698349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183052024-02-20T22:48:44.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774711469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183062024-02-20T22:48:44.276Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774724798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183072024-02-20T22:48:44.276Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774744398Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183082024-02-20T22:48:44.276Z{"msg":"7c172915-8586-4b16-9916-c8a0b8cb5e10 is now active with session: 76a44ea2-ddf3-439f-877e-d2d14d320ed5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774757727Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183092024-02-20T22:48:44.276Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774769717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818,"session_id":"76a44ea2-ddf3-439f-877e-d2d14d320ed5"}
183102024-02-20T22:48:44.276Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:43.774788936Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818}
183112024-02-20T22:48:44.276ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183122024-02-20T22:48:44.276ZLoading write count information from file "/tmp/test_repair_verify.data"
183132024-02-20T22:48:44.276ZRun Repair workload
183142024-02-20T22:48:44.276Z01/30 Write block 202 len 8192 data: 8 10
183152024-02-20T22:48:44.284Z02/30 Write block 82 len 16384 data: 5 6 6 6
183162024-02-20T22:48:44.284Z03/30 Write block 407 len 36864 data: 14 12 14 15 15 16 19 20 18
183172024-02-20T22:48:44.284Z04/30 Write block 528 len 24576 data: 11 12 14 14 10 12
183182024-02-20T22:48:44.284Z05/30 Read block 80 len 20480
183192024-02-20T22:48:44.284Z06/30 Write block 513 len 8192 data: 12 12
183202024-02-20T22:48:44.284Z07/30 Write block 285 len 28672 data: 14 13 14 13 9 8 9
183212024-02-20T22:48:44.284Z08/30 Flush
183222024-02-20T22:48:44.285Z09/30 Write block 22 len 4096 data: 13
183232024-02-20T22:48:44.285Z10/30 Read block 591 len 20480
183242024-02-20T22:48:44.285Z11/30 Write block 412 len 40960 data: 17 20 21 19 15 13 14 17 17 13
183252024-02-20T22:48:44.285Z12/30 Write block 305 len 4096 data: 19
183262024-02-20T22:48:44.285Z13/30 Write block 174 len 12288 data: 11 14 13
183272024-02-20T22:48:44.285Z14/30 Flush
183282024-02-20T22:48:44.289Z15/30 Write block 185 len 36864 data: 11 11 13 11 11 11 12 12 11
183292024-02-20T22:48:44.289Z16/30 Read block 27 len 40960
183302024-02-20T22:48:44.290Z17/30 Read block 347 len 24576
183312024-02-20T22:48:44.290Z18/30 Write block 171 len 32768 data: 11 11 10 12 15 14 14 14
183322024-02-20T22:48:44.290Z19/30 Read block 506 len 12288
183332024-02-20T22:48:44.290Z20/30 Read block 427 len 4096
183342024-02-20T22:48:44.294Z21/30 Read block 367 len 12288
183352024-02-20T22:48:44.294Z22/30 Write block 415 len 16384 data: 20 16 14 15
183362024-02-20T22:48:44.294Z23/30 Read block 155 len 16384
183372024-02-20T22:48:44.294Z24/30 Read block 347 len 8192
183382024-02-20T22:48:44.294Z25/30 Write block 303 len 4096 data: 20
183392024-02-20T22:48:44.294Z26/30 Read block 129 len 40960
183402024-02-20T22:48:44.299Z27/30 Write block 176 len 28672 data: 15 15 15 17 16 19 17
183412024-02-20T22:48:44.299Z28/30 Write block 356 len 4096 data: 11
183422024-02-20T22:48:44.299Z29/30 Write block 336 len 4096 data: 14
183432024-02-20T22:48:44.299Z30/30 Read block 456 len 24576
183442024-02-20T22:48:44.299Z----------------------------------------------------------------
183452024-02-20T22:48:44.299Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183462024-02-20T22:48:44.299ZStates: Active Active Active
183472024-02-20T22:48:44.300ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183482024-02-20T22:48:44.300Z 1 Acked 1000 Write 2 Done Done Done false
183492024-02-20T22:48:44.300Z 2 Acked 1001 Write 4 Done Sent Done false
183502024-02-20T22:48:44.300Z 3 Acked 1002 Write 9 Done Sent Done false
183512024-02-20T22:48:44.300Z 4 Acked 1003 Write 6 Done Sent Done false
183522024-02-20T22:48:44.300Z 5 Acked 1004 Read 5 Done Sent Done false
183532024-02-20T22:48:44.300Z 6 Acked 1005 Write 2 Done Sent Done false
183542024-02-20T22:48:44.300Z 7 Acked 1006 Write 7 Done Sent Done false
183552024-02-20T22:48:44.300Z 8 Acked 1007 Flush 0 Done Sent Done false
183562024-02-20T22:48:44.300Z 9 Acked 1008 Write 1 Done Sent Done false
183572024-02-20T22:48:44.300Z 10 Acked 1009 Read 5 Done Sent Done false
183582024-02-20T22:48:44.300Z 11 Acked 1010 Write 10 Done Sent Done false
183592024-02-20T22:48:44.300Z 12 Acked 1011 Write 1 Done Sent Done false
183602024-02-20T22:48:44.300Z 13 Acked 1012 Write 3 Done Sent Done false
183612024-02-20T22:48:44.300Z 14 Acked 1013 Flush 0 Done Sent Done false
183622024-02-20T22:48:44.300Z 15 Acked 1014 Write 9 Done Sent Done false
183632024-02-20T22:48:44.300Z 16 Acked 1015 Read 10 Done Sent Done false
183642024-02-20T22:48:44.300Z 17 Acked 1016 Read 6 Done Sent Done false
183652024-02-20T22:48:44.300Z 18 Acked 1017 Write 8 Done Sent Done false
183662024-02-20T22:48:44.300Z 19 Acked 1018 Read 3 Done Sent Done false
183672024-02-20T22:48:44.300Z 20 Acked 1019 Read 1 Done Sent Done false
183682024-02-20T22:48:44.300Z 21 Acked 1020 Read 3 Done Sent Done false
183692024-02-20T22:48:44.301Z 22 Acked 1021 Write 4 Done Sent Done false
183702024-02-20T22:48:44.301Z 23 Acked 1022 Read 4 Done Sent Done false
183712024-02-20T22:48:44.301Z 24 Acked 1023 Read 2 Done Sent Done false
183722024-02-20T22:48:44.301Z 25 Acked 1024 Write 1 Done Sent Done false
183732024-02-20T22:48:44.301Z 26 Acked 1025 Read 10 Done Sent Done false
183742024-02-20T22:48:44.301Z 27 Acked 1026 Write 7 Done Sent Done false
183752024-02-20T22:48:44.301Z 28 Acked 1027 Write 1 Done Sent Done false
183762024-02-20T22:48:44.301Z 29 Acked 1028 Write 1 Done Sent Done false
183772024-02-20T22:48:44.301Z 30 Acked 1029 Read 6 Sent Sent Done false
183782024-02-20T22:48:44.301Z STATES DS:0 DS:1 DS:2 TOTAL
183792024-02-20T22:48:44.301Z New 0 0 0 0
183802024-02-20T22:48:44.301Z Sent 1 29 0 30
183812024-02-20T22:48:44.301Z Done 29 1 30 60
183822024-02-20T22:48:44.301Z Skipped 0 0 0 0
183832024-02-20T22:48:44.301Z Error 0 0 0 0
183842024-02-20T22:48:44.301ZLast Flush: 1013 0 1013
183852024-02-20T22:48:44.301ZDownstairs last five completed:
183862024-02-20T22:48:44.301ZUpstairs last five completed: 30 29 28 27 26
183872024-02-20T22:48:44.301Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:43.804216173Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1818}
183882024-02-20T22:48:44.301ZWrote out file "/tmp/test_repair_verify.data"
183892024-02-20T22:48:44.306Z
183902024-02-20T22:48:44.306ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183912024-02-20T22:48:44.321ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183922024-02-20T22:48:44.321Z 0 000-019 166 166 166 281 281 281 F F F
183932024-02-20T22:48:44.321Z 1 020-039 170 167 170 292 284 292 F F F <---
183942024-02-20T22:48:44.321Z 2 040-059 169 169 169 290 290 290 F F F
183952024-02-20T22:48:44.321Z 3 060-079 168 168 168 286 286 286 F F F
183962024-02-20T22:48:44.321Z 4 080-099 170 164 170 291 279 291 F F F <---
183972024-02-20T22:48:44.321Z 5 100-119 168 168 168 286 286 286 F F F
183982024-02-20T22:48:44.326Z 6 120-139 166 166 166 282 282 282 F F F
183992024-02-20T22:48:44.326Z 7 140-159 168 168 168 289 289 289 F F F
184002024-02-20T22:48:44.326Z 8 160-179 170 164 170 292 279 292 T F T <---
184012024-02-20T22:48:44.326Z 9 180-199 158 158 158 271 271 271 T F T <---
184022024-02-20T22:48:44.326Z 10 200-219 170 168 170 291 288 291 F T F <---
184032024-02-20T22:48:44.326Z 11 220-239 162 162 162 276 276 276 F F F
184042024-02-20T22:48:44.326Z 12 240-259 168 168 168 287 287 287 F F F
184052024-02-20T22:48:44.326Z 13 260-279 163 163 163 278 278 278 F F F
184062024-02-20T22:48:44.327Z 14 280-299 170 164 170 291 279 291 F F F <---
184072024-02-20T22:48:44.327Z 15 300-319 170 162 170 292 276 292 T F T <---
184082024-02-20T22:48:44.327Z 16 320-339 165 165 165 280 280 280 T F T <---
184092024-02-20T22:48:44.327Z 17 340-359 163 163 163 278 278 278 T F T <---
184102024-02-20T22:48:44.327Z 18 360-379 168 168 168 288 288 288 F F F
184112024-02-20T22:48:44.327Z 19 380-399 168 168 168 288 288 288 F F F
184122024-02-20T22:48:44.327Z 20 400-419 170 168 170 292 286 292 T F T <---
184132024-02-20T22:48:44.327Z 21 420-439 170 168 170 292 286 292 F F F <---
184142024-02-20T22:48:44.327Z 22 440-459 168 168 168 286 286 286 F F F
184152024-02-20T22:48:44.327Z 23 460-479 168 168 168 288 288 288 F F F
184162024-02-20T22:48:44.327Z 24 480-499 168 168 168 288 288 288 F F F
184172024-02-20T22:48:44.327Z 25 500-519 170 159 170 291 272 291 F F F <---
184182024-02-20T22:48:44.327Z 26 520-539 170 168 170 291 287 291 F F F <---
184192024-02-20T22:48:44.327Z 27 540-559 166 166 166 281 281 281 F F F
184202024-02-20T22:48:44.327Z 28 560-579 166 166 166 281 281 281 F F F
184212024-02-20T22:48:44.327Z 29 580-599 168 168 168 286 286 286 F F F
184222024-02-20T22:48:44.327ZMax gen: 170, Max flush: 292
184232024-02-20T22:48:44.327ZError: Difference in extent metadata found!
184242024-02-20T22:48:44.327ZA Difference in extent metadata is expected here
184252024-02-20T22:48:44.327ZOn loop 85
184262024-02-20T22:48:44.327Z
184272024-02-20T22:48:44.332ZVerifying data now
184282024-02-20T22:48:48.974ZNo change after verify
184292024-02-20T22:48:48.974ZLoop: 85 Downstairs dump after verify (and repair):
184302024-02-20T22:48:48.991ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184312024-02-20T22:48:48.991Z 0 000-019 166 166 166 281 281 281 F F F
184322024-02-20T22:48:48.991Z 1 020-039 170 170 170 292 292 292 F F F
184332024-02-20T22:48:48.991Z 2 040-059 169 169 169 290 290 290 F F F
184342024-02-20T22:48:48.991Z 3 060-079 168 168 168 286 286 286 F F F
184352024-02-20T22:48:48.991Z 4 080-099 170 170 170 291 291 291 F F F
184362024-02-20T22:48:48.991Z 5 100-119 168 168 168 286 286 286 F F F
184372024-02-20T22:48:48.991Z 6 120-139 166 166 166 282 282 282 F F F
184382024-02-20T22:48:48.991Z 7 140-159 168 168 168 289 289 289 F F F
184392024-02-20T22:48:48.992Z 8 160-179 171 171 171 293 293 293 F F F
184402024-02-20T22:48:48.992Z 9 180-199 171 171 171 293 293 293 F F F
184412024-02-20T22:48:48.992Z 10 200-219 170 170 170 291 291 291 F F F
184422024-02-20T22:48:48.992Z 11 220-239 162 162 162 276 276 276 F F F
184432024-02-20T22:48:48.992Z 12 240-259 168 168 168 287 287 287 F F F
184442024-02-20T22:48:48.992Z 13 260-279 163 163 163 278 278 278 F F F
184452024-02-20T22:48:48.992Z 14 280-299 170 170 170 291 291 291 F F F
184462024-02-20T22:48:48.992Z 15 300-319 171 171 171 293 293 293 F F F
184472024-02-20T22:48:48.992Z 16 320-339 171 171 171 293 293 293 F F F
184482024-02-20T22:48:48.992Z 17 340-359 171 171 171 293 293 293 F F F
184492024-02-20T22:48:48.992Z 18 360-379 168 168 168 288 288 288 F F F
184502024-02-20T22:48:48.992Z 19 380-399 168 168 168 288 288 288 F F F
184512024-02-20T22:48:48.992Z 20 400-419 171 171 171 293 293 293 F F F
184522024-02-20T22:48:48.993Z 21 420-439 170 170 170 292 292 292 F F F
184532024-02-20T22:48:48.993Z 22 440-459 168 168 168 286 286 286 F F F
184542024-02-20T22:48:48.993Z 23 460-479 168 168 168 288 288 288 F F F
184552024-02-20T22:48:48.993Z 24 480-499 168 168 168 288 288 288 F F F
184562024-02-20T22:48:48.993Z 25 500-519 170 170 170 291 291 291 F F F
184572024-02-20T22:48:48.993Z 26 520-539 170 170 170 291 291 291 F F F
184582024-02-20T22:48:48.993Z 27 540-559 166 166 166 281 281 281 F F F
184592024-02-20T22:48:48.993Z 28 560-579 166 166 166 281 281 281 F F F
184602024-02-20T22:48:48.993Z 29 580-599 168 168 168 286 286 286 F F F
184612024-02-20T22:48:48.993ZMax gen: 171, Max flush: 293
184622024-02-20T22:48:49.006Z
184632024-02-20T22:48:49.006ZBegin loop 86
184642024-02-20T22:48:49.006ZDownstairs to restart: 0
184652024-02-20T22:48:49.133Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514500465Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826}
184662024-02-20T22:48:49.133Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514734417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826}
184672024-02-20T22:48:49.133Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514755956Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826}
184682024-02-20T22:48:49.133Z{"msg":"Crucible 683a4a30-6c7a-4214-aa83-b3554f9884de has session id: 61a9de55-550c-4c50-b3d3-04ee73e7bbb9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514788285Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184692024-02-20T22:48:49.133Z{"msg":"Upstairs opts: Upstairs UUID: 683a4a30-6c7a-4214-aa83-b3554f9884de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
184702024-02-20T22:48:49.133Ztime":"2024-02-20T22:48:48.514833504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184712024-02-20T22:48:49.133Z{"msg":"Crucible stats registered with UUID: 683a4a30-6c7a-4214-aa83-b3554f9884de","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514880842Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184722024-02-20T22:48:49.133Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514919941Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826}
184732024-02-20T22:48:49.133Z{"msg":"683a4a30-6c7a-4214-aa83-b3554f9884de active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.514988978Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184742024-02-20T22:48:49.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515032697Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184752024-02-20T22:48:49.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515059885Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184762024-02-20T22:48:49.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515093814Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184772024-02-20T22:48:49.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515120063Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184782024-02-20T22:48:49.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515145642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184792024-02-20T22:48:49.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515182561Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184802024-02-20T22:48:49.133Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.51520877Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184812024-02-20T22:48:49.133Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515251859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184822024-02-20T22:48:49.133Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515279128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184832024-02-20T22:48:49.133Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515444892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184842024-02-20T22:48:49.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.51549415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184852024-02-20T22:48:49.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.515522069Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184862024-02-20T22:48:49.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.516224884Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184872024-02-20T22:48:49.134Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.516267432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184882024-02-20T22:48:49.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.516307971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184892024-02-20T22:48:49.134Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.5163419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184902024-02-20T22:48:49.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.516371058Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184912024-02-20T22:48:49.134Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.516421297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184922024-02-20T22:48:49.134Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.516995406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184932024-02-20T22:48:49.134Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517035205Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184942024-02-20T22:48:49.134Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517057824Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184952024-02-20T22:48:49.134Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517477849Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184962024-02-20T22:48:49.134Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517536217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184972024-02-20T22:48:49.134Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517590225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184982024-02-20T22:48:49.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517612054Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
184992024-02-20T22:48:49.135Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517641383Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185002024-02-20T22:48:49.135Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517657112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185012024-02-20T22:48:49.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517677612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185022024-02-20T22:48:49.135Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517691991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185032024-02-20T22:48:49.135Z{"msg":"[0]R flush_numbers[0..12]: [281, 292, 290, 286, 291, 286, 282, 289, 293, 293, 291, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517713981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185042024-02-20T22:48:49.135Z{"msg":"[0]R generation[0..12]: [166, 170, 169, 168, 170, 168, 166, 168, 171, 171, 170, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.51772919Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185052024-02-20T22:48:49.135Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517755839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185062024-02-20T22:48:49.135Z{"msg":"[1]R flush_numbers[0..12]: [281, 292, 290, 286, 291, 286, 282, 289, 293, 293, 291, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517776328Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185072024-02-20T22:48:49.135Z{"msg":"[1]R generation[0..12]: [166, 170, 169, 168, 170, 168, 166, 168, 171, 171, 170, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517806637Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185082024-02-20T22:48:49.135Z{"msg":"Wait for a query_work_queue command to finish before sending IO
185092024-02-20T22:48:49.135Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517830656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185102024-02-20T22:48:49.135Z{"msg":"[2]R flush_numbers[0..12]: [281, 292, 290, 286, 291, 286, 282, 289, 293, 293, 291, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517845546Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185112024-02-20T22:48:49.135Z{"msg":"[2]R generation[0..12]: [166, 170, 169, 168, 170, 168, 166, 168, 171, 171, 170, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517867495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185122024-02-20T22:48:49.135Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517882144Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185132024-02-20T22:48:49.135Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517896134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185142024-02-20T22:48:49.135Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517909754Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185152024-02-20T22:48:49.136Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517923633Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185162024-02-20T22:48:49.136Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517937233Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185172024-02-20T22:48:49.136Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517950612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185182024-02-20T22:48:49.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517962692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185192024-02-20T22:48:49.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.517983251Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185202024-02-20T22:48:49.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.51799695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185212024-02-20T22:48:49.136Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.51801034Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185222024-02-20T22:48:49.136Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.518029639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185232024-02-20T22:48:49.136Z{"msg":"683a4a30-6c7a-4214-aa83-b3554f9884de is now active with session: 61a9de55-550c-4c50-b3d3-04ee73e7bbb9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.518042939Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185242024-02-20T22:48:49.136Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.518055808Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826,"session_id":"61a9de55-550c-4c50-b3d3-04ee73e7bbb9"}
185252024-02-20T22:48:49.136Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:48.518077157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826}
185262024-02-20T22:48:49.136ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185272024-02-20T22:48:49.136ZLoading write count information from file "/tmp/test_repair_verify.data"
185282024-02-20T22:48:49.136ZRun Repair workload
185292024-02-20T22:48:49.136Z01/30 Write block 511 len 20480 data: 16 11 13 13 13
185302024-02-20T22:48:49.136Z02/30 Read block 448 len 20480
185312024-02-20T22:48:49.136Z03/30 Read block 464 len 12288
185322024-02-20T22:48:49.136Z04/30 Write block 87 len 4096 data: 7
185332024-02-20T22:48:49.136Z05/30 Write block 164 len 32768 data: 7 10 11 9 10 10 12 12
185342024-02-20T22:48:49.137Z06/30 Read block 168 len 20480
185352024-02-20T22:48:49.146Z07/30 Write block 80 len 40960 data: 9 6 6 7 7 7 6 8 9 9
185362024-02-20T22:48:49.146Z08/30 Read block 470 len 36864
185372024-02-20T22:48:49.146Z09/30 Read block 13 len 4096
185382024-02-20T22:48:49.146Z10/30 Write block 112 len 16384 data: 13 14 13 11
185392024-02-20T22:48:49.146Z11/30 Read block 69 len 24576
185402024-02-20T22:48:49.146Z12/30 Read block 176 len 16384
185412024-02-20T22:48:49.147Z13/30 Read block 401 len 8192
185422024-02-20T22:48:49.147Z14/30 Write block 368 len 8192 data: 13 16
185432024-02-20T22:48:49.147Z15/30 Read block 95 len 32768
185442024-02-20T22:48:49.147Z16/30 Flush
185452024-02-20T22:48:49.147Z17/30 Write block 561 len 12288 data: 11 8 9
185462024-02-20T22:48:49.147Z18/30 Read block 201 len 32768
185472024-02-20T22:48:49.147Z19/30 Read block 569 len 24576
185482024-02-20T22:48:49.147Z20/30 Read block 166 len 20480
185492024-02-20T22:48:49.147Z21/30 Read block 328 len 4096
185502024-02-20T22:48:49.148Z22/30 Read block 568 len 16384
185512024-02-20T22:48:49.148Z23/30 Write block 371 len 36864 data: 22 19 16 13 12 13 17 14 13
185522024-02-20T22:48:49.148Z24/30 Flush
185532024-02-20T22:48:49.148Z25/30 Write block 529 len 28672 data: 13 15 15 11 13 15 19
185542024-02-20T22:48:49.148Z26/30 Read block 3 len 12288
185552024-02-20T22:48:49.148Z27/30 Write block 271 len 12288 data: 9 9 7
185562024-02-20T22:48:49.148Z28/30 Write block 596 len 12288 data: 14 13 12
185572024-02-20T22:48:49.148Z29/30 Read block 361 len 40960
185582024-02-20T22:48:49.148Z30/30 Write block 476 len 4096 data: 15
185592024-02-20T22:48:49.148Z----------------------------------------------------------------
185602024-02-20T22:48:49.149Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185612024-02-20T22:48:49.149ZStates: Active Active Active
185622024-02-20T22:48:49.149ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185632024-02-20T22:48:49.149Z 1 Acked 1000 Write 5 Sent Done Done false
185642024-02-20T22:48:49.149Z 2 Acked 1001 Read 5 Sent Done Done false
185652024-02-20T22:48:49.149Z 3 Acked 1002 Read 3 Sent Done Done false
185662024-02-20T22:48:49.149Z 4 Acked 1003 Write 1 Sent Done Done false
185672024-02-20T22:48:49.149Z 5 Acked 1004 Write 8 Sent Done Done false
185682024-02-20T22:48:49.149Z 6 Acked 1005 Read 5 Sent Done Done false
185692024-02-20T22:48:49.149Z 7 Acked 1006 Write 10 Sent Done Done false
185702024-02-20T22:48:49.149Z 8 Acked 1007 Read 9 Sent Done Done false
185712024-02-20T22:48:49.149Z 9 Acked 1008 Read 1 Sent Done Done false
185722024-02-20T22:48:49.150Z 10 Acked 1009 Write 4 Sent Done Done false
185732024-02-20T22:48:49.150Z 11 Acked 1010 Read 6 Sent Done Done false
185742024-02-20T22:48:49.150Z 12 Acked 1011 Read 4 Sent Done Done false
185752024-02-20T22:48:49.150Z 13 Acked 1012 Read 2 Sent Done Done false
185762024-02-20T22:48:49.150Z 14 Acked 1013 Write 2 Sent Done Done false
185772024-02-20T22:48:49.150Z 15 Acked 1014 Read 8 Sent Done Done false
185782024-02-20T22:48:49.150Z 16 Acked 1015 Flush 0 Sent Done Done false
185792024-02-20T22:48:49.151Z 17 Acked 1016 Write 3 Sent Done Done false
185802024-02-20T22:48:49.151Z 18 Acked 1017 Read 8 Sent Done Done false
185812024-02-20T22:48:49.151Z 19 Acked 1018 Read 6 Sent Done Done false
185822024-02-20T22:48:49.151Z 20 Acked 1019 Read 5 Sent Done Done false
185832024-02-20T22:48:49.151Z 21 Acked 1020 Read 1 Sent Done Done false
185842024-02-20T22:48:49.151Z 22 Acked 1021 Read 4 Sent Done Done false
185852024-02-20T22:48:49.151Z 23 Acked 1022 Write 9 Sent Done Done false
185862024-02-20T22:48:49.151Z 24 Acked 1023 Flush 0 Sent Done Done false
185872024-02-20T22:48:49.151Z 25 Acked 1024 Write 7 Sent Done Done false
185882024-02-20T22:48:49.152Z 26 Acked 1025 Read 3 Sent Done Done false
185892024-02-20T22:48:49.152Z 27 Acked 1026 Write 3 Sent Done Done false
185902024-02-20T22:48:49.152Z 28 Acked 1027 Write 3 Sent Done Done false
185912024-02-20T22:48:49.152Z 29 Acked 1028 Read 10 Sent Done Done false
185922024-02-20T22:48:49.152Z 30 Acked 1029 Write 1 Sent Sent Sent false
185932024-02-20T22:48:49.152Z STATES DS:0 DS:1 DS:2 TOTAL
185942024-02-20T22:48:49.152Z New 0 0 0 0
185952024-02-20T22:48:49.152Z Sent 30 1 1 32
185962024-02-20T22:48:49.152Z Done 0 29 29 58
185972024-02-20T22:48:49.152Z Skipped 0 0 0 0
185982024-02-20T22:48:49.152Z Error 0 0 0 0
185992024-02-20T22:48:49.153ZLast Flush: 0 1023 1023
186002024-02-20T22:48:49.153ZDownstairs last five completed:
186012024-02-20T22:48:49.153ZUpstairs last five completed: 30 29 28 27 26
186022024-02-20T22:48:49.153Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:48.548065695Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1826}
186032024-02-20T22:48:49.153ZWrote out file "/tmp/test_repair_verify.data"
186042024-02-20T22:48:49.153Z
186052024-02-20T22:48:49.153ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186062024-02-20T22:48:49.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186072024-02-20T22:48:49.154Z 0 000-019 166 166 166 281 281 281 F F F
186082024-02-20T22:48:49.154Z 1 020-039 170 170 170 292 292 292 F F F
186092024-02-20T22:48:49.154Z 2 040-059 169 169 169 290 290 290 F F F
186102024-02-20T22:48:49.154Z 3 060-079 168 168 168 286 286 286 F F F
186112024-02-20T22:48:49.154Z 4 080-099 170 172 172 291 294 294 F F F <---
186122024-02-20T22:48:49.154Z 5 100-119 168 172 172 286 294 294 F F F <---
186132024-02-20T22:48:49.154Z 6 120-139 166 166 166 282 282 282 F F F
186142024-02-20T22:48:49.154Z 7 140-159 168 168 168 289 289 289 F F F
186152024-02-20T22:48:49.154Z 8 160-179 171 172 172 293 294 294 F F F <---
186162024-02-20T22:48:49.154Z 9 180-199 171 171 171 293 293 293 F F F
186172024-02-20T22:48:49.154Z 10 200-219 170 170 170 291 291 291 F F F
186182024-02-20T22:48:49.155Z 11 220-239 162 162 162 276 276 276 F F F
186192024-02-20T22:48:49.155Z 12 240-259 168 168 168 287 287 287 F F F
186202024-02-20T22:48:49.155Z 13 260-279 163 163 163 278 278 278 F T T <---
186212024-02-20T22:48:49.155Z 14 280-299 170 170 170 291 291 291 F F F
186222024-02-20T22:48:49.155Z 15 300-319 171 171 171 293 293 293 F F F
186232024-02-20T22:48:49.155Z 16 320-339 171 171 171 293 293 293 F F F
186242024-02-20T22:48:49.155Z 17 340-359 171 171 171 293 293 293 F F F
186252024-02-20T22:48:49.155Z 18 360-379 168 172 172 288 295 295 F F F <---
186262024-02-20T22:48:49.155Z 19 380-399 168 168 168 288 288 288 F F F
186272024-02-20T22:48:49.156Z 20 400-419 171 171 171 293 293 293 F F F
186282024-02-20T22:48:49.156Z 21 420-439 170 170 170 292 292 292 F F F
186292024-02-20T22:48:49.156Z 22 440-459 168 168 168 286 286 286 F F F
186302024-02-20T22:48:49.156Z 23 460-479 168 168 168 288 288 288 F T T <---
186312024-02-20T22:48:49.156Z 24 480-499 168 168 168 288 288 288 F F F
186322024-02-20T22:48:49.156Z 25 500-519 170 172 172 291 294 294 F F F <---
186332024-02-20T22:48:49.156Z 26 520-539 170 170 170 291 291 291 F T T <---
186342024-02-20T22:48:49.156Z 27 540-559 166 166 166 281 281 281 F F F
186352024-02-20T22:48:49.162Z 28 560-579 166 172 172 281 295 295 F F F <---
186362024-02-20T22:48:49.162Z 29 580-599 168 168 168 286 286 286 F T T <---
186372024-02-20T22:48:49.162ZMax gen: 172, Max flush: 295
186382024-02-20T22:48:49.162ZError: Difference in extent metadata found!
186392024-02-20T22:48:49.162ZA Difference in extent metadata is expected here
186402024-02-20T22:48:49.163ZOn loop 86
186412024-02-20T22:48:49.163Z
186422024-02-20T22:48:49.163ZVerifying data now
186432024-02-20T22:48:53.589ZNo change after verify
186442024-02-20T22:48:53.589ZLoop: 86 Downstairs dump after verify (and repair):
186452024-02-20T22:48:53.603ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186462024-02-20T22:48:53.603Z 0 000-019 166 166 166 281 281 281 F F F
186472024-02-20T22:48:53.603Z 1 020-039 170 170 170 292 292 292 F F F
186482024-02-20T22:48:53.603Z 2 040-059 169 169 169 290 290 290 F F F
186492024-02-20T22:48:53.603Z 3 060-079 168 168 168 286 286 286 F F F
186502024-02-20T22:48:53.603Z 4 080-099 172 172 172 294 294 294 F F F
186512024-02-20T22:48:53.603Z 5 100-119 172 172 172 294 294 294 F F F
186522024-02-20T22:48:53.603Z 6 120-139 166 166 166 282 282 282 F F F
186532024-02-20T22:48:53.604Z 7 140-159 168 168 168 289 289 289 F F F
186542024-02-20T22:48:53.608Z 8 160-179 172 172 172 294 294 294 F F F
186552024-02-20T22:48:53.608Z 9 180-199 171 171 171 293 293 293 F F F
186562024-02-20T22:48:53.608Z 10 200-219 170 170 170 291 291 291 F F F
186572024-02-20T22:48:53.608Z 11 220-239 162 162 162 276 276 276 F F F
186582024-02-20T22:48:53.608Z 12 240-259 168 168 168 287 287 287 F F F
186592024-02-20T22:48:53.608Z 13 260-279 173 173 173 296 296 296 F F F
186602024-02-20T22:48:53.609Z 14 280-299 170 170 170 291 291 291 F F F
186612024-02-20T22:48:53.609Z 15 300-319 171 171 171 293 293 293 F F F
186622024-02-20T22:48:53.609Z 16 320-339 171 171 171 293 293 293 F F F
186632024-02-20T22:48:53.609Z 17 340-359 171 171 171 293 293 293 F F F
186642024-02-20T22:48:53.609Z 18 360-379 172 172 172 295 295 295 F F F
186652024-02-20T22:48:53.609Z 19 380-399 168 168 168 288 288 288 F F F
186662024-02-20T22:48:53.609Z 20 400-419 171 171 171 293 293 293 F F F
186672024-02-20T22:48:53.609Z 21 420-439 170 170 170 292 292 292 F F F
186682024-02-20T22:48:53.609Z 22 440-459 168 168 168 286 286 286 F F F
186692024-02-20T22:48:53.609Z 23 460-479 173 173 173 296 296 296 F F F
186702024-02-20T22:48:53.609Z 24 480-499 168 168 168 288 288 288 F F F
186712024-02-20T22:48:53.609Z 25 500-519 172 172 172 294 294 294 F F F
186722024-02-20T22:48:53.609Z 26 520-539 173 173 173 296 296 296 F F F
186732024-02-20T22:48:53.609Z 27 540-559 166 166 166 281 281 281 F F F
186742024-02-20T22:48:53.609Z 28 560-579 172 172 172 295 295 295 F F F
186752024-02-20T22:48:53.609Z 29 580-599 173 173 173 296 296 296 F F F
186762024-02-20T22:48:53.609ZMax gen: 173, Max flush: 296
186772024-02-20T22:48:53.609Z
186782024-02-20T22:48:53.609ZBegin loop 87
186792024-02-20T22:48:53.609ZDownstairs to restart: 2
186802024-02-20T22:48:53.625Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13107313Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834}
186812024-02-20T22:48:53.625Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131394979Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834}
186822024-02-20T22:48:53.625Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
186832024-02-20T22:48:53.636Z,"time":"2024-02-20T22:48:53.131436887Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834}
186842024-02-20T22:48:53.636Z{"msg":"Crucible 0e02f106-5f4f-4874-adea-0e987ab55f70 has session id: cc356467-ab11-4b66-831a-a28d5eabd3f8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131475886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186852024-02-20T22:48:53.636Z{"msg":"Upstairs opts: Upstairs UUID: 0e02f106-5f4f-4874-adea-0e987ab55f70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131523314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186862024-02-20T22:48:53.636Z{"msg":"Crucible stats registered with UUID: 0e02f106-5f4f-4874-adea-0e987ab55f70","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131570432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186872024-02-20T22:48:53.636Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13162671Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834}
186882024-02-20T22:48:53.636Z{"msg":"0e02f106-5f4f-4874-adea-0e987ab55f70 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13165112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186892024-02-20T22:48:53.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131692648Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186902024-02-20T22:48:53.637Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131723397Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186912024-02-20T22:48:53.637Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131767235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186922024-02-20T22:48:53.637Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131800784Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186932024-02-20T22:48:53.637Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131841373Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186942024-02-20T22:48:53.637Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131871492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186952024-02-20T22:48:53.637Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13191615Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186962024-02-20T22:48:53.637Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.131960389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186972024-02-20T22:48:53.637Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132001197Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186982024-02-20T22:48:53.637Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132028906Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
186992024-02-20T22:48:53.637Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132064615Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187002024-02-20T22:48:53.637Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132180901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187012024-02-20T22:48:53.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132683033Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187022024-02-20T22:48:53.637Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132726731Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187032024-02-20T22:48:53.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132742461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187042024-02-20T22:48:53.637Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132785099Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187052024-02-20T22:48:53.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132799908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187062024-02-20T22:48:53.637Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.132897385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187072024-02-20T22:48:53.637Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.133447125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187082024-02-20T22:48:53.637Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.133493024Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187092024-02-20T22:48:53.637Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.133523222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187102024-02-20T22:48:53.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134017195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187112024-02-20T22:48:53.637Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134048604Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187122024-02-20T22:48:53.637Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134065283Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187132024-02-20T22:48:53.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134112192Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187142024-02-20T22:48:53.637Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13413846Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187152024-02-20T22:48:53.638Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13415333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187162024-02-20T22:48:53.638Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134173209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187172024-02-20T22:48:53.638Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134203088Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187182024-02-20T22:48:53.638Z{"msg":"[0]R flush_numbers[0..12]: [281, 292, 290, 286, 294, 294, 282, 289, 294, 293, 291, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134219638Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187192024-02-20T22:48:53.638Z{"msg":"[0]R generation[0..12]: [166, 170, 169, 168, 172, 172, 166, 168, 172, 171, 170, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134244227Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187202024-02-20T22:48:53.638Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134268426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187212024-02-20T22:48:53.638Z{"msg":"[1]R flush_numbers[0..12]: [281, 292, 290, 286, 294, 294, 282, 289, 294, 293, 291, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134293235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187222024-02-20T22:48:53.638Z{"msg":"[1]R generation[0..12]: [166, 170, 169, 168, 172, 172, 166, 168, 172, 171, 170, 162]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
187232024-02-20T22:48:53.638Z,"time":"2024-02-20T22:48:53.134324874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187242024-02-20T22:48:53.638Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134351973Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187252024-02-20T22:48:53.638Z{"msg":"[2]R flush_numbers[0..12]: [281, 292, 290, 286, 294, 294, 282, 289, 294, 293, 291, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134380732Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187262024-02-20T22:48:53.638Z{"msg":"[2]R generation[0..12]: [166, 170, 169, 168, 172, 172, 166, 168, 172, 171, 170, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134407651Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187272024-02-20T22:48:53.638Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134446509Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187282024-02-20T22:48:53.638Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134473719Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187292024-02-20T22:48:53.638Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134501018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187302024-02-20T22:48:53.638Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134525747Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187312024-02-20T22:48:53.638Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134561576Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187322024-02-20T22:48:53.638Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134589415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187332024-02-20T22:48:53.638Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134616974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187342024-02-20T22:48:53.638Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134631573Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187352024-02-20T22:48:53.638Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187362024-02-20T22:48:53.638Z,"level":30,"time":"2024-02-20T22:48:53.134667412Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187372024-02-20T22:48:53.638Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.13470338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187382024-02-20T22:48:53.638Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134737179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187392024-02-20T22:48:53.639Z{"msg":"0e02f106-5f4f-4874-adea-0e987ab55f70 is now active with session: cc356467-ab11-4b66-831a-a28d5eabd3f8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134770168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187402024-02-20T22:48:53.639Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134800307Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834,"session_id":"cc356467-ab11-4b66-831a-a28d5eabd3f8"}
187412024-02-20T22:48:53.639Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:53.134831276Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834}
187422024-02-20T22:48:53.639ZLoading write count information from file "/tmp/test_repair_verify.data"
187432024-02-20T22:48:53.639ZRun Repair workload
187442024-02-20T22:48:53.639Z01/30 Write block 93 len 28672 data: 10 12 11 8 12 8 13
187452024-02-20T22:48:53.644Z02/30 Write block 137 len 16384 data: 9 9 10 12
187462024-02-20T22:48:53.644Z03/30 Write block 515 len 24576 data: 14 12 15 14 11 14
187472024-02-20T22:48:53.644Z04/30 Write block 184 len 24576 data: 15 12 12 14 12 12
187482024-02-20T22:48:53.644Z05/30 Read block 529 len 4096
187492024-02-20T22:48:53.644Z06/30 Write block 596 len 4096 data: 15
187502024-02-20T22:48:53.644Z07/30 Write block 181 len 40960 data: 20 18 14 16 13 13 15 13 13 12
187512024-02-20T22:48:53.644Z08/30 Read block 404 len 32768
187522024-02-20T22:48:53.644Z09/30 Write block 0 len 32768 data: 2 4 9 12 14 14 9 7
187532024-02-20T22:48:53.644Z10/30 Write block 44 len 16384 data: 11 8 9 10
187542024-02-20T22:48:53.644Z11/30 Write block 486 len 12288 data: 14 15 14
187552024-02-20T22:48:53.645Z12/30 Write block 11 len 28672 data: 10 9 11 10 11 15 13
187562024-02-20T22:48:53.645Z13/30 Write block 580 len 8192 data: 20 17
187572024-02-20T22:48:53.645Z14/30 Flush
187582024-02-20T22:48:53.649Z15/30 Write block 552 len 8192 data: 7 9
187592024-02-20T22:48:53.649Z16/30 Read block 332 len 24576
187602024-02-20T22:48:53.649Z17/30 Flush
187612024-02-20T22:48:53.653Z18/30 Write block 380 len 40960 data: 13 10 13 12 13 11 9 8 9 13
187622024-02-20T22:48:53.653Z19/30 Write block 162 len 40960 data: 9 8 8 11 12 10 11 11 13 13
187632024-02-20T22:48:53.653Z20/30 Read block 533 len 24576
187642024-02-20T22:48:53.653Z21/30 Write block 503 len 32768 data: 8 11 13 12 13 12 12 12
187652024-02-20T22:48:53.653Z22/30 Write block 361 len 16384 data: 17 20 22 20
187662024-02-20T22:48:53.657Z23/30 Write block 442 len 20480 data: 15 12 11 14 13
187672024-02-20T22:48:53.657Z24/30 Flush
187682024-02-20T22:48:53.657Z25/30 Write block 26 len 12288 data: 15 20 20
187692024-02-20T22:48:53.661Z26/30 Read block 118 len 36864
187702024-02-20T22:48:53.661Z27/30 Write block 76 len 28672 data: 7 8 9 8 10 7 7
187712024-02-20T22:48:53.661Z28/30 Write block 204 len 28672 data: 11 12 14 14 13 14 17
187722024-02-20T22:48:53.661Z29/30 Write block 115 len 16384 data: 12 14 15 11
187732024-02-20T22:48:53.662Z30/30 Write block 75 len 8192 data: 8 8
187742024-02-20T22:48:53.666Z----------------------------------------------------------------
187752024-02-20T22:48:53.666Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187762024-02-20T22:48:53.666ZStates: Active Active Active
187772024-02-20T22:48:53.666ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187782024-02-20T22:48:53.666Z 1 Acked 1000 Write 7 Done Done Done false
187792024-02-20T22:48:53.666Z 2 Acked 1001 Write 4 Done Done Done false
187802024-02-20T22:48:53.666Z 3 Acked 1002 Write 6 Done Done Sent false
187812024-02-20T22:48:53.666Z 4 Acked 1003 Write 6 Done Done Sent false
187822024-02-20T22:48:53.666Z 5 Acked 1004 Read 1 Done Done Sent false
187832024-02-20T22:48:53.666Z 6 Acked 1005 Write 1 Done Done Sent false
187842024-02-20T22:48:53.666Z 7 Acked 1006 Write 10 Done Done Sent false
187852024-02-20T22:48:53.666Z 8 Acked 1007 Read 8 Done Done Sent false
187862024-02-20T22:48:53.666Z 9 Acked 1008 Write 8 Done Done Sent false
187872024-02-20T22:48:53.666Z 10 Acked 1009 Write 4 Done Done Sent false
187882024-02-20T22:48:53.666Z 11 Acked 1010 Write 3 Done Done Sent false
187892024-02-20T22:48:53.666Z 12 Acked 1011 Write 7 Done Done Sent false
187902024-02-20T22:48:53.666Z 13 Acked 1012 Write 2 Done Done Sent false
187912024-02-20T22:48:53.666Z 14 Acked 1013 Flush 0 Done Done Sent false
187922024-02-20T22:48:53.666Z 15 Acked 1014 Write 2 Done Done Sent false
187932024-02-20T22:48:53.666Z 16 Acked 1015 Read 6 Done Done Sent false
187942024-02-20T22:48:53.667Z 17 Acked 1016 Flush 0 Done Done Sent false
187952024-02-20T22:48:53.667Z 18 Acked 1017 Write 10 Done Done Sent false
187962024-02-20T22:48:53.667Z 19 Acked 1018 Write 10 Done Done Sent false
187972024-02-20T22:48:53.667Z 20 Acked 1019 Read 6 Done Done Sent false
187982024-02-20T22:48:53.667Z 21 Acked 1020 Write 8 Done Done Sent false
187992024-02-20T22:48:53.667Z 22 Acked 1021 Write 4 Done Done Sent false
188002024-02-20T22:48:53.667Z 23 Acked 1022 Write 5 Done Done Sent false
188012024-02-20T22:48:53.667Z 24 Acked 1023 Flush 0 Done Done Sent false
188022024-02-20T22:48:53.667Z 25 Acked 1024 Write 3 Done Done Sent false
188032024-02-20T22:48:53.667Z 26 Acked 1025 Read 9 Done Done Sent false
188042024-02-20T22:48:53.667Z 27 Acked 1026 Write 7 Done Done Sent false
188052024-02-20T22:48:53.667Z 28 Acked 1027 Write 7 Done Done Sent false
188062024-02-20T22:48:53.667Z 29 Acked 1028 Write 4 Done Done Sent false
188072024-02-20T22:48:53.667Z 30 Acked 1029 Write 2 Sent Sent Sent false
188082024-02-20T22:48:53.667Z STATES DS:0 DS:1 DS:2 TOTAL
188092024-02-20T22:48:53.667Z New 0 0 0 0
188102024-02-20T22:48:53.667Z Sent 1 1 28 30
188112024-02-20T22:48:53.667Z Done 29 29 2 60
188122024-02-20T22:48:53.667Z Skipped 0 0 0 0
188132024-02-20T22:48:53.667Z Error 0 0 0 0
188142024-02-20T22:48:53.667ZLast Flush: 1023 1023 0
188152024-02-20T22:48:53.667ZDownstairs last five completed:
188162024-02-20T22:48:53.667ZUpstairs last five completed: 30 29 28 27 26
188172024-02-20T22:48:53.667Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:53.168279989Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1834}
188182024-02-20T22:48:53.667ZWrote out file "/tmp/test_repair_verify.data"
188192024-02-20T22:48:53.672Z
188202024-02-20T22:48:53.672ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188212024-02-20T22:48:53.686ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188222024-02-20T22:48:53.686Z 0 000-019 174 174 166 297 297 281 F F F <---
188232024-02-20T22:48:53.686Z 1 020-039 170 170 170 292 292 292 T T F <---
188242024-02-20T22:48:53.686Z 2 040-059 174 174 169 297 297 290 F F F <---
188252024-02-20T22:48:53.686Z 3 060-079 168 168 168 286 286 286 T T F <---
188262024-02-20T22:48:53.686Z 4 080-099 174 174 172 297 297 294 T T T <---
188272024-02-20T22:48:53.686Z 5 100-119 172 172 172 294 294 294 T T F <---
188282024-02-20T22:48:53.686Z 6 120-139 174 174 166 297 297 282 F F T <---
188292024-02-20T22:48:53.691Z 7 140-159 174 174 168 297 297 289 F F T <---
188302024-02-20T22:48:53.691Z 8 160-179 174 174 172 299 299 294 F F F <---
188312024-02-20T22:48:53.691Z 9 180-199 174 174 171 297 297 293 F F F <---
188322024-02-20T22:48:53.691Z 10 200-219 170 170 170 291 291 291 T T F <---
188332024-02-20T22:48:53.691Z 11 220-239 162 162 162 276 276 276 F F F
188342024-02-20T22:48:53.691Z 12 240-259 168 168 168 287 287 287 F F F
188352024-02-20T22:48:53.691Z 13 260-279 173 173 173 296 296 296 F F F
188362024-02-20T22:48:53.691Z 14 280-299 170 170 170 291 291 291 F F F
188372024-02-20T22:48:53.691Z 15 300-319 171 171 171 293 293 293 F F F
188382024-02-20T22:48:53.691Z 16 320-339 171 171 171 293 293 293 F F F
188392024-02-20T22:48:53.691Z 17 340-359 171 171 171 293 293 293 F F F
188402024-02-20T22:48:53.691Z 18 360-379 174 174 172 299 299 295 F F F <---
188412024-02-20T22:48:53.691Z 19 380-399 174 174 168 299 299 288 F F F <---
188422024-02-20T22:48:53.691Z 20 400-419 171 171 171 293 293 293 F F F
188432024-02-20T22:48:53.691Z 21 420-439 170 170 170 292 292 292 F F F
188442024-02-20T22:48:53.691Z 22 440-459 174 174 168 299 299 286 F F F <---
188452024-02-20T22:48:53.691Z 23 460-479 173 173 173 296 296 296 F F F
188462024-02-20T22:48:53.691Z 24 480-499 174 174 168 297 297 288 F F F <---
188472024-02-20T22:48:53.692Z 25 500-519 174 174 172 299 299 294 F F F <---
188482024-02-20T22:48:53.692Z 26 520-539 174 174 173 297 297 296 F F F <---
188492024-02-20T22:48:53.692Z 27 540-559 174 174 166 298 298 281 F F F <---
188502024-02-20T22:48:53.692Z 28 560-579 172 172 172 295 295 295 F F F
188512024-02-20T22:48:53.692Z 29 580-599 174 174 173 297 297 296 F F F <---
188522024-02-20T22:48:53.692ZMax gen: 174, Max flush: 299
188532024-02-20T22:48:53.692ZError: Difference in extent metadata found!
188542024-02-20T22:48:53.692ZA Difference in extent metadata is expected here
188552024-02-20T22:48:53.692ZOn loop 87
188562024-02-20T22:48:53.692Z
188572024-02-20T22:48:53.696ZVerifying data now
188582024-02-20T22:48:59.577ZNo change after verify
188592024-02-20T22:48:59.577ZLoop: 87 Downstairs dump after verify (and repair):
188602024-02-20T22:48:59.589ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188612024-02-20T22:48:59.589Z 0 000-019 174 174 174 297 297 297 F F F
188622024-02-20T22:48:59.589Z 1 020-039 175 175 175 300 300 300 F F F
188632024-02-20T22:48:59.590Z 2 040-059 174 174 174 297 297 297 F F F
188642024-02-20T22:48:59.590Z 3 060-079 175 175 175 300 300 300 F F F
188652024-02-20T22:48:59.590Z 4 080-099 175 175 175 300 300 300 F F F
188662024-02-20T22:48:59.594Z 5 100-119 175 175 175 300 300 300 F F F
188672024-02-20T22:48:59.594Z 6 120-139 174 174 174 297 297 297 F F F
188682024-02-20T22:48:59.594Z 7 140-159 174 174 174 297 297 297 F F F
188692024-02-20T22:48:59.594Z 8 160-179 174 174 174 299 299 299 F F F
188702024-02-20T22:48:59.594Z 9 180-199 174 174 174 297 297 297 F F F
188712024-02-20T22:48:59.594Z 10 200-219 175 175 175 300 300 300 F F F
188722024-02-20T22:48:59.594Z 11 220-239 162 162 162 276 276 276 F F F
188732024-02-20T22:48:59.594Z 12 240-259 168 168 168 287 287 287 F F F
188742024-02-20T22:48:59.594Z 13 260-279 173 173 173 296 296 296 F F F
188752024-02-20T22:48:59.595Z 14 280-299 170 170 170 291 291 291 F F F
188762024-02-20T22:48:59.595Z 15 300-319 171 171 171 293 293 293 F F F
188772024-02-20T22:48:59.595Z 16 320-339 171 171 171 293 293 293 F F F
188782024-02-20T22:48:59.595Z 17 340-359 171 171 171 293 293 293 F F F
188792024-02-20T22:48:59.595Z 18 360-379 174 174 174 299 299 299 F F F
188802024-02-20T22:48:59.595Z 19 380-399 174 174 174 299 299 299 F F F
188812024-02-20T22:48:59.595Z 20 400-419 171 171 171 293 293 293 F F F
188822024-02-20T22:48:59.595Z 21 420-439 170 170 170 292 292 292 F F F
188832024-02-20T22:48:59.595Z 22 440-459 174 174 174 299 299 299 F F F
188842024-02-20T22:48:59.595Z 23 460-479 173 173 173 296 296 296 F F F
188852024-02-20T22:48:59.595Z 24 480-499 174 174 174 297 297 297 F F F
188862024-02-20T22:48:59.595Z 25 500-519 174 174 174 299 299 299 F F F
188872024-02-20T22:48:59.595Z 26 520-539 174 174 174 297 297 297 F F F
188882024-02-20T22:48:59.596Z 27 540-559 174 174 174 298 298 298 F F F
188892024-02-20T22:48:59.596Z 28 560-579 172 172 172 295 295 295 F F F
188902024-02-20T22:48:59.596Z 29 580-599 174 174 174 297 297 297 F F F
188912024-02-20T22:48:59.596ZMax gen: 175, Max flush: 300
188922024-02-20T22:48:59.596Z
188932024-02-20T22:48:59.596ZBegin loop 88
188942024-02-20T22:48:59.596ZDownstairs to restart: 2
188952024-02-20T22:48:59.611Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.11716285Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842}
188962024-02-20T22:48:59.611Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117426811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842}
188972024-02-20T22:48:59.621Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.11745102Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842}
188982024-02-20T22:48:59.621Z{"msg":"Crucible 5712f5dd-ab39-4300-a104-5c4fe2345d83 has session id: f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117483249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
188992024-02-20T22:48:59.621Z{"msg":"Crucible runtime is spawned
189002024-02-20T22:48:59.621ZUpstairs opts: Upstairs UUID: 5712f5dd-ab39-4300-a104-5c4fe2345d83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117524577Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189012024-02-20T22:48:59.621Z{"msg":"Crucible stats registered with UUID: 5712f5dd-ab39-4300-a104-5c4fe2345d83","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117560726Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189022024-02-20T22:48:59.621Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117589575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842}
189032024-02-20T22:48:59.621Z{"msg":"5712f5dd-ab39-4300-a104-5c4fe2345d83 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.11771799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189042024-02-20T22:48:59.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117752619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189052024-02-20T22:48:59.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117799287Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189062024-02-20T22:48:59.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117846365Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189072024-02-20T22:48:59.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117883464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189082024-02-20T22:48:59.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117928402Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189092024-02-20T22:48:59.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.117957821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189102024-02-20T22:48:59.622Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.11798466Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189112024-02-20T22:48:59.623Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118035229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189122024-02-20T22:48:59.623Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118064278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189132024-02-20T22:48:59.623Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118101436Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189142024-02-20T22:48:59.623Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118127826Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189152024-02-20T22:48:59.623Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118154145Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189162024-02-20T22:48:59.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118791602Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189172024-02-20T22:48:59.623Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.11883283Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189182024-02-20T22:48:59.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.1188525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189192024-02-20T22:48:59.623Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118891448Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189202024-02-20T22:48:59.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118918067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189212024-02-20T22:48:59.624Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.118956116Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189222024-02-20T22:48:59.624Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.119484607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189232024-02-20T22:48:59.624Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.119524996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189242024-02-20T22:48:59.624Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.119562834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189252024-02-20T22:48:59.624Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120049707Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189262024-02-20T22:48:59.625Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120093415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189272024-02-20T22:48:59.625Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120111595Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189282024-02-20T22:48:59.625Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120184302Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189292024-02-20T22:48:59.625Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120212301Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189302024-02-20T22:48:59.625Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.12022723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189312024-02-20T22:48:59.625Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120295518Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189322024-02-20T22:48:59.625Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120331727Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189332024-02-20T22:48:59.626Z{"msg":"[0]R flush_numbers[0..12]: [297, 300, 297, 300, 300, 300, 297, 297, 299, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120358866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189342024-02-20T22:48:59.626Z{"msg":"[0]R generation[0..12]: [174, 175, 174, 175, 175, 175, 174, 174, 174, 174, 175, 162]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
189352024-02-20T22:48:59.626Z,"time":"2024-02-20T22:48:59.120391964Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189362024-02-20T22:48:59.626Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120418013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189372024-02-20T22:48:59.626Z{"msg":"[1]R flush_numbers[0..12]: [297, 300, 297, 300, 300, 300, 297, 297, 299, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120446623Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189382024-02-20T22:48:59.627Z{"msg":"[1]R generation[0..12]: [174, 175, 174, 175, 175, 175, 174, 174, 174, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120479831Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189392024-02-20T22:48:59.627Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.1205137Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189402024-02-20T22:48:59.627Z{"msg":"[2]R flush_numbers[0..12]: [297, 300, 297, 300, 300, 300, 297, 297, 299, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120540879Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189412024-02-20T22:48:59.627Z{"msg":"[2]R generation[0..12]: [174, 175, 174, 175, 175, 175, 174, 174, 174, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120573818Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189422024-02-20T22:48:59.627Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120600337Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189432024-02-20T22:48:59.628Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120631266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189442024-02-20T22:48:59.628Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120655645Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189452024-02-20T22:48:59.628Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120681644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189462024-02-20T22:48:59.628Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120704683Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189472024-02-20T22:48:59.628Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120732312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189482024-02-20T22:48:59.629Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120754401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189492024-02-20T22:48:59.629Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120782191Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189502024-02-20T22:48:59.629Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189512024-02-20T22:48:59.629Z,"time":"2024-02-20T22:48:59.12080557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189522024-02-20T22:48:59.629Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120840558Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189532024-02-20T22:48:59.630Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120871677Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189542024-02-20T22:48:59.630Z{"msg":"5712f5dd-ab39-4300-a104-5c4fe2345d83 is now active with session: f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120896016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189552024-02-20T22:48:59.630Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120921875Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842,"session_id":"f9a4b43e-a6ef-4459-b9bf-9d8f558e94c3"}
189562024-02-20T22:48:59.630Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:48:59.120943775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842}
189572024-02-20T22:48:59.630ZLoading write count information from file "/tmp/test_repair_verify.data"
189582024-02-20T22:48:59.630ZRun Repair workload
189592024-02-20T22:49:00.048Z01/30 Write block 577 len 24576 data: 16 19 22 21 18 17
189602024-02-20T22:49:00.048Z02/30 Write block 527 len 32768 data: 13 12 14 16 16 12 14 16
189612024-02-20T22:49:00.048Z03/30 Write block 98 len 24576 data: 9 14 12 11 10 9
189622024-02-20T22:49:00.048Z04/30 Read block 533 len 4096
189632024-02-20T22:49:00.048Z05/30 Write block 494 len 20480 data: 14 12 13 14 12
189642024-02-20T22:49:00.048Z06/30 Write block 467 len 32768 data: 15 18 21 16 13 12 11 12
189652024-02-20T22:49:00.048Z07/30 Write block 492 len 20480 data: 13 15 15 13 14
189662024-02-20T22:49:00.048Z08/30 Read block 211 len 4096
189672024-02-20T22:49:00.048Z09/30 Read block 499 len 8192
189682024-02-20T22:49:00.048Z10/30 Write block 293 len 28672 data: 7 6 5 8 10 13 14
189692024-02-20T22:49:00.048Z11/30 Write block 513 len 28672 data: 14 14 15 13 16 15 12
189702024-02-20T22:49:00.048Z12/30 Write block 389 len 28672 data: 14 13 13 15 14 12 10
189712024-02-20T22:49:00.048Z13/30 Write block 360 len 36864 data: 17 18 21 23 21 21 20 13 14
189722024-02-20T22:49:00.048Z14/30 Write block 290 len 12288 data: 9 10 7
189732024-02-20T22:49:00.048Z15/30 Read block 390 len 32768
189742024-02-20T22:49:00.048Z16/30 Read block 114 len 20480
189752024-02-20T22:49:00.048Z17/30 Write block 577 len 32768 data: 17 20 23 22 19 18 16 10
189762024-02-20T22:49:00.049Z18/30 Write block 101 len 32768 data: 12 11 10 11 10 10 10 13
189772024-02-20T22:49:00.049Z19/30 Read block 375 len 40960
189782024-02-20T22:49:00.049Z20/30 Read block 113 len 20480
189792024-02-20T22:49:00.049Z21/30 Read block 267 len 20480
189802024-02-20T22:49:00.049Z22/30 Write block 491 len 28672 data: 11 14 16 16 14 15 15
189812024-02-20T22:49:00.049Z23/30 Read block 567 len 20480
189822024-02-20T22:49:00.055Z24/30 Write block 327 len 32768 data: 18 17 15 12 11 16 17 15
189832024-02-20T22:49:00.055Z25/30 Flush
189842024-02-20T22:49:00.055Z26/30 Write block 368 len 8192 data: 15 17
189852024-02-20T22:49:00.055Z27/30 Write block 483 len 28672 data: 19 17 16 15 16 15 12
189862024-02-20T22:49:00.055Z28/30 Read block 144 len 40960
189872024-02-20T22:49:00.055Z29/30 Read block 121 len 24576
189882024-02-20T22:49:00.055Z30/30 Write block 369 len 12288 data: 18 19 23
189892024-02-20T22:49:00.055Z----------------------------------------------------------------
189902024-02-20T22:49:00.055Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
189912024-02-20T22:49:00.055ZStates: Active Active Active
189922024-02-20T22:49:00.055ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189932024-02-20T22:49:00.056Z 1 Acked 1000 Write 6 Done Done Done false
189942024-02-20T22:49:00.056Z 2 Acked 1001 Write 8 Done Done Done false
189952024-02-20T22:49:00.056Z 3 Acked 1002 Write 6 Done Done Done false
189962024-02-20T22:49:00.056Z 4 Acked 1003 Read 1 Done Done Done false
189972024-02-20T22:49:00.056Z 5 Acked 1004 Write 5 Done Done Done false
189982024-02-20T22:49:00.056Z 6 Acked 1005 Write 8 Done Done Done false
189992024-02-20T22:49:00.056Z 7 Acked 1006 Write 5 Done Done Done false
190002024-02-20T22:49:00.056Z 8 Acked 1007 Read 1 Done Done Done false
190012024-02-20T22:49:00.056Z 9 Acked 1008 Read 2 Done Done Sent false
190022024-02-20T22:49:00.056Z 10 Acked 1009 Write 7 Done Done Sent false
190032024-02-20T22:49:00.056Z 11 Acked 1010 Write 7 Done Done Sent false
190042024-02-20T22:49:00.056Z 12 Acked 1011 Write 7 Done Done Sent false
190052024-02-20T22:49:00.056Z 13 Acked 1012 Write 9 Done Done Sent false
190062024-02-20T22:49:00.056Z 14 Acked 1013 Write 3 Done Done Sent false
190072024-02-20T22:49:00.056Z 15 Acked 1014 Read 8 Done Done Sent false
190082024-02-20T22:49:00.056Z 16 Acked 1015 Read 5 Done Done Sent false
190092024-02-20T22:49:00.056Z 17 Acked 1016 Write 8 Done Done Sent false
190102024-02-20T22:49:00.056Z 18 Acked 1017 Write 8 Done Done Sent false
190112024-02-20T22:49:00.056Z 19 Acked 1018 Read 10 Done Done Sent false
190122024-02-20T22:49:00.056Z 20 Acked 1019 Read 5 Done Done Sent false
190132024-02-20T22:49:00.056Z 21 Acked 1020 Read 5 Done Done Sent false
190142024-02-20T22:49:00.056Z 22 Acked 1021 Write 7 Done Done Sent false
190152024-02-20T22:49:00.056Z 23 Acked 1022 Read 5 Done Done Sent false
190162024-02-20T22:49:00.056Z 24 Acked 1023 Write 8 Done Done Sent false
190172024-02-20T22:49:00.057Z 25 Acked 1024 Flush 0 Done Done Sent false
190182024-02-20T22:49:00.057Z 26 Acked 1025 Write 2 Done Done Sent false
190192024-02-20T22:49:00.057Z 27 Acked 1026 Write 7 Done Done Sent false
190202024-02-20T22:49:00.057Z 28 Acked 1027 Read 10 Done Done Sent false
190212024-02-20T22:49:00.057Z 29 Acked 1028 Read 6 Done Done Sent false
190222024-02-20T22:49:00.057Z 30 Acked 1029 Write 3 Sent Sent Sent false
190232024-02-20T22:49:00.057Z STATES DS:0 DS:1 DS:2 TOTAL
190242024-02-20T22:49:00.057Z New 0 0 0 0
190252024-02-20T22:49:00.057Z Sent 1 1 22 24
190262024-02-20T22:49:00.057Z Done 29 29 8 66
190272024-02-20T22:49:00.057Z Skipped 0 0 0 0
190282024-02-20T22:49:00.057Z Error 0 0 0 0
190292024-02-20T22:49:00.057ZLast Flush: 1024 1024 0
190302024-02-20T22:49:00.057ZDownstairs last five completed:
190312024-02-20T22:49:00.057ZUpstairs last five completed: 30 29 28 27 26
190322024-02-20T22:49:00.057Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:48:59.153263188Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1842}
190332024-02-20T22:49:00.057ZWrote out file "/tmp/test_repair_verify.data"
190342024-02-20T22:49:00.057Z
190352024-02-20T22:49:00.057ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190362024-02-20T22:49:00.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190372024-02-20T22:49:00.057Z 0 000-019 174 174 174 297 297 297 F F F
190382024-02-20T22:49:00.057Z 1 020-039 175 175 175 300 300 300 F F F
190392024-02-20T22:49:00.057Z 2 040-059 174 174 174 297 297 297 F F F
190402024-02-20T22:49:00.057Z 3 060-079 175 175 175 300 300 300 F F F
190412024-02-20T22:49:00.057Z 4 080-099 176 176 175 301 301 300 F F T <---
190422024-02-20T22:49:00.057Z 5 100-119 176 176 175 301 301 300 F F T <---
190432024-02-20T22:49:00.058Z 6 120-139 174 174 174 297 297 297 F F F
190442024-02-20T22:49:00.058Z 7 140-159 174 174 174 297 297 297 F F F
190452024-02-20T22:49:00.058Z 8 160-179 174 174 174 299 299 299 F F F
190462024-02-20T22:49:00.058Z 9 180-199 174 174 174 297 297 297 F F F
190472024-02-20T22:49:00.058Z 10 200-219 175 175 175 300 300 300 F F F
190482024-02-20T22:49:00.058Z 11 220-239 162 162 162 276 276 276 F F F
190492024-02-20T22:49:00.058Z 12 240-259 168 168 168 287 287 287 F F F
190502024-02-20T22:49:00.058Z 13 260-279 173 173 173 296 296 296 F F F
190512024-02-20T22:49:00.058Z 14 280-299 176 176 170 301 301 291 F F F <---
190522024-02-20T22:49:00.058Z 15 300-319 171 171 171 293 293 293 F F F
190532024-02-20T22:49:00.058Z 16 320-339 176 176 171 301 301 293 F F F <---
190542024-02-20T22:49:00.058Z 17 340-359 171 171 171 293 293 293 F F F
190552024-02-20T22:49:00.058Z 18 360-379 176 176 174 301 301 299 T T F <---
190562024-02-20T22:49:00.058Z 19 380-399 176 176 174 301 301 299 F F F <---
190572024-02-20T22:49:00.058Z 20 400-419 171 171 171 293 293 293 F F F
190582024-02-20T22:49:00.058Z 21 420-439 170 170 170 292 292 292 F F F
190592024-02-20T22:49:00.058Z 22 440-459 174 174 174 299 299 299 F F F
190602024-02-20T22:49:00.058Z 23 460-479 176 176 173 301 301 296 F F T <---
190612024-02-20T22:49:00.058Z 24 480-499 176 176 174 301 301 297 T T T <---
190622024-02-20T22:49:00.058Z 25 500-519 176 176 174 301 301 299 F F F <---
190632024-02-20T22:49:00.058Z 26 520-539 176 176 174 301 301 297 F F T <---
190642024-02-20T22:49:00.058Z 27 540-559 174 174 174 298 298 298 F F F
190652024-02-20T22:49:00.058Z 28 560-579 176 176 172 301 301 295 F F T <---
190662024-02-20T22:49:00.058Z 29 580-599 176 176 174 301 301 297 F F T <---
190672024-02-20T22:49:00.058ZMax gen: 176, Max flush: 301
190682024-02-20T22:49:00.059ZError: Difference in extent metadata found!
190692024-02-20T22:49:00.059ZA Difference in extent metadata is expected here
190702024-02-20T22:49:00.059ZOn loop 88
190712024-02-20T22:49:00.059Z
190722024-02-20T22:49:00.059ZVerifying data now
190732024-02-20T22:49:05.260ZNo change after verify
190742024-02-20T22:49:05.260ZLoop: 88 Downstairs dump after verify (and repair):
190752024-02-20T22:49:05.273ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190762024-02-20T22:49:05.273Z 0 000-019 174 174 174 297 297 297 F F F
190772024-02-20T22:49:05.273Z 1 020-039 175 175 175 300 300 300 F F F
190782024-02-20T22:49:05.273Z 2 040-059 174 174 174 297 297 297 F F F
190792024-02-20T22:49:05.273Z 3 060-079 175 175 175 300 300 300 F F F
190802024-02-20T22:49:05.273Z 4 080-099 176 176 176 301 301 301 F F F
190812024-02-20T22:49:05.277Z 5 100-119 176 176 176 301 301 301 F F F
190822024-02-20T22:49:05.277Z 6 120-139 174 174 174 297 297 297 F F F
190832024-02-20T22:49:05.277Z 7 140-159 174 174 174 297 297 297 F F F
190842024-02-20T22:49:05.277Z 8 160-179 174 174 174 299 299 299 F F F
190852024-02-20T22:49:05.277Z 9 180-199 174 174 174 297 297 297 F F F
190862024-02-20T22:49:05.277Z 10 200-219 175 175 175 300 300 300 F F F
190872024-02-20T22:49:05.277Z 11 220-239 162 162 162 276 276 276 F F F
190882024-02-20T22:49:05.277Z 12 240-259 168 168 168 287 287 287 F F F
190892024-02-20T22:49:05.277Z 13 260-279 173 173 173 296 296 296 F F F
190902024-02-20T22:49:05.277Z 14 280-299 176 176 176 301 301 301 F F F
190912024-02-20T22:49:05.278Z 15 300-319 171 171 171 293 293 293 F F F
190922024-02-20T22:49:05.278Z 16 320-339 176 176 176 301 301 301 F F F
190932024-02-20T22:49:05.278Z 17 340-359 171 171 171 293 293 293 F F F
190942024-02-20T22:49:05.278Z 18 360-379 177 177 177 302 302 302 F F F
190952024-02-20T22:49:05.278Z 19 380-399 176 176 176 301 301 301 F F F
190962024-02-20T22:49:05.278Z 20 400-419 171 171 171 293 293 293 F F F
190972024-02-20T22:49:05.278Z 21 420-439 170 170 170 292 292 292 F F F
190982024-02-20T22:49:05.278Z 22 440-459 174 174 174 299 299 299 F F F
190992024-02-20T22:49:05.278Z 23 460-479 176 176 176 301 301 301 F F F
191002024-02-20T22:49:05.278Z 24 480-499 177 177 177 302 302 302 F F F
191012024-02-20T22:49:05.278Z 25 500-519 176 176 176 301 301 301 F F F
191022024-02-20T22:49:05.278Z 26 520-539 176 176 176 301 301 301 F F F
191032024-02-20T22:49:05.278Z 27 540-559 174 174 174 298 298 298 F F F
191042024-02-20T22:49:05.278Z 28 560-579 176 176 176 301 301 301 F F F
191052024-02-20T22:49:05.278Z 29 580-599 176 176 176 301 301 301 F F F
191062024-02-20T22:49:05.278ZMax gen: 177, Max flush: 302
191072024-02-20T22:49:05.278Z
191082024-02-20T22:49:05.278ZBegin loop 89
191092024-02-20T22:49:05.278ZDownstairs to restart: 0
191102024-02-20T22:49:05.294Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798445616Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850}
191112024-02-20T22:49:05.294Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798726537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850}
191122024-02-20T22:49:05.294Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798751966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850}
191132024-02-20T22:49:05.302Z{"msg":"Crucible f4faf85a-23d0-4f54-b97c-de47f25597da has session id: 14b254a7-4818-46dc-a043-bae664e0d0d4","v":0,"name":"crucible","level":30Crucible runtime is spawned
191142024-02-20T22:49:05.302Z,"time":"2024-02-20T22:49:04.798788444Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191152024-02-20T22:49:05.303Z{"msg":"Upstairs opts: Upstairs UUID: f4faf85a-23d0-4f54-b97c-de47f25597da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798825123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191162024-02-20T22:49:05.303Z{"msg":"Crucible stats registered with UUID: f4faf85a-23d0-4f54-b97c-de47f25597da","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798858832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191172024-02-20T22:49:05.303Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798875261Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850}
191182024-02-20T22:49:05.303Z{"msg":"f4faf85a-23d0-4f54-b97c-de47f25597da active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.798985067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191192024-02-20T22:49:05.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799012286Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191202024-02-20T22:49:05.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799040125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191212024-02-20T22:49:05.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799065545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191222024-02-20T22:49:05.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799093623Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191232024-02-20T22:49:05.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799138162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191242024-02-20T22:49:05.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.79920074Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191252024-02-20T22:49:05.303Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799236678Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191262024-02-20T22:49:05.303Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799276147Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191272024-02-20T22:49:05.303Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799302646Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191282024-02-20T22:49:05.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799433961Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191292024-02-20T22:49:05.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.79947043Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191302024-02-20T22:49:05.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.799514798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191312024-02-20T22:49:05.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.800234112Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191322024-02-20T22:49:05.303Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.800269711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191332024-02-20T22:49:05.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.800285901Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191342024-02-20T22:49:05.303Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.80030721Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191352024-02-20T22:49:05.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.800321649Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191362024-02-20T22:49:05.303Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.800335419Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191372024-02-20T22:49:05.303Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.800979746Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191382024-02-20T22:49:05.303Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801032804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191392024-02-20T22:49:05.303Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801078162Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191402024-02-20T22:49:05.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801578235Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191412024-02-20T22:49:05.304Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801622243Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191422024-02-20T22:49:05.304Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801639312Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191432024-02-20T22:49:05.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801666511Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191442024-02-20T22:49:05.304Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801684711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191452024-02-20T22:49:05.304Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.8017081Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191462024-02-20T22:49:05.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801722799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191472024-02-20T22:49:05.304Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801747958Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191482024-02-20T22:49:05.304Z{"msg":"[0]R flush_numbers[0..12]: [297, 300, 297, 300, 301, 301, 297, 297, 299, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801767418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191492024-02-20T22:49:05.304Z{"msg":"[0]R generation[0..12]: [174, 175, 174, 175, 176, 176, 174, 174, 174, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801790647Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191502024-02-20T22:49:05.304Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801811146Z","hostname":"Wait for a query_work_queue command to finish before sending IO
191512024-02-20T22:49:05.304Zip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191522024-02-20T22:49:05.304Z{"msg":"[1]R flush_numbers[0..12]: [297, 300, 297, 300, 301, 301, 297, 297, 299, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801850215Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191532024-02-20T22:49:05.304Z{"msg":"[1]R generation[0..12]: [174, 175, 174, 175, 176, 176, 174, 174, 174, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801882634Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191542024-02-20T22:49:05.304Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801917292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191552024-02-20T22:49:05.304Z{"msg":"[2]R flush_numbers[0..12]: [297, 300, 297, 300, 301, 301, 297, 297, 299, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.801950661Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191562024-02-20T22:49:05.304Z{"msg":"[2]R generation[0..12]: [174, 175, 174, 175, 176, 176, 174, 174, 174, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.80198751Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191572024-02-20T22:49:05.304Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802014719Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191582024-02-20T22:49:05.304Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802065767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191592024-02-20T22:49:05.304Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802100586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191602024-02-20T22:49:05.304Z{"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802121125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191612024-02-20T22:49:05.304Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802153204Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191622024-02-20T22:49:05.304Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802168943Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191632024-02-20T22:49:05.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802189963Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191642024-02-20T22:49:05.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802214362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191652024-02-20T22:49:05.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802239291Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191662024-02-20T22:49:05.305Zhostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191672024-02-20T22:49:05.305Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.80226886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191682024-02-20T22:49:05.305Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802286659Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191692024-02-20T22:49:05.305Z{"msg":"f4faf85a-23d0-4f54-b97c-de47f25597da is now active with session: 14b254a7-4818-46dc-a043-bae664e0d0d4","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802315938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191702024-02-20T22:49:05.305Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802331157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850,"session_id":"14b254a7-4818-46dc-a043-bae664e0d0d4"}
191712024-02-20T22:49:05.305Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:04.802361016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850}
191722024-02-20T22:49:05.305ZLoading write count information from file "/tmp/test_repair_verify.data"
191732024-02-20T22:49:05.305ZRun Repair workload
191742024-02-20T22:49:05.309Z01/30 Write block 413 len 24576 data: 21 22 21 17 15 16
191752024-02-20T22:49:05.310Z02/30 Read block 280 len 4096
191762024-02-20T22:49:05.310Z03/30 Read block 220 len 16384
191772024-02-20T22:49:05.310Z04/30 Write block 525 len 36864 data: 10 12 14 13 15 17 17 13 15
191782024-02-20T22:49:05.310Z05/30 Write block 529 len 20480 data: 16 18 18 14 16
191792024-02-20T22:49:05.310Z06/30 Read block 182 len 8192
191802024-02-20T22:49:05.310Z07/30 Flush
191812024-02-20T22:49:05.310Z08/30 Write block 403 len 16384 data: 16 17 15 15
191822024-02-20T22:49:05.310Z09/30 Read block 41 len 4096
191832024-02-20T22:49:05.310Z10/30 Write block 364 len 24576 data: 22 22 21 14 16 19
191842024-02-20T22:49:05.310Z11/30 Write block 41 len 36864 data: 12 11 12 12 9 10 11 13 14
191852024-02-20T22:49:05.310Z12/30 Read block 460 len 28672
191862024-02-20T22:49:05.314Z13/30 Read block 188 len 4096
191872024-02-20T22:49:05.314Z14/30 Write block 465 len 16384 data: 17 12 16 19
191882024-02-20T22:49:05.314Z15/30 Write block 488 len 28672 data: 16 13 11 12 15 17 17
191892024-02-20T22:49:05.314Z16/30 Read block 373 len 16384
191902024-02-20T22:49:05.314Z17/30 Flush
191912024-02-20T22:49:05.318Z18/30 Write block 278 len 16384 data: 8 8 10 12
191922024-02-20T22:49:05.318Z19/30 Read block 551 len 20480
191932024-02-20T22:49:05.318Z20/30 Write block 39 len 12288 data: 17 13 13
191942024-02-20T22:49:05.322Z21/30 Write block 349 len 24576 data: 12 13 10 8 7 6
191952024-02-20T22:49:05.322Z22/30 Read block 155 len 24576
191962024-02-20T22:49:05.322Z23/30 Write block 350 len 32768 data: 14 11 9 8 7 7 12 7
191972024-02-20T22:49:05.322Z24/30 Read block 149 len 40960
191982024-02-20T22:49:05.326Z25/30 Read block 252 len 36864
191992024-02-20T22:49:05.326Z26/30 Flush
192002024-02-20T22:49:05.330Z27/30 Write block 16 len 12288 data: 16 14 14
192012024-02-20T22:49:05.330Z28/30 Write block 160 len 32768 data: 12 9 10 9 9 12 13 11
192022024-02-20T22:49:05.330Z29/30 Read block 77 len 24576
192032024-02-20T22:49:05.334Z30/30 Write block 34 len 32768 data: 24 25 20 19 17 18 14 14
192042024-02-20T22:49:05.334Z----------------------------------------------------------------
192052024-02-20T22:49:05.334Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
192062024-02-20T22:49:05.334ZStates: Active Active Active
192072024-02-20T22:49:05.334ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192082024-02-20T22:49:05.334Z 1 Acked 1000 Write 6 Sent Done Done false
192092024-02-20T22:49:05.334Z 2 Acked 1001 Read 1 Sent Done Done false
192102024-02-20T22:49:05.335Z 3 Acked 1002 Read 4 Sent Done Done false
192112024-02-20T22:49:05.335Z 4 Acked 1003 Write 9 Sent Done Done false
192122024-02-20T22:49:05.335Z 5 Acked 1004 Write 5 Sent Done Done false
192132024-02-20T22:49:05.335Z 6 Acked 1005 Read 2 Sent Done Done false
192142024-02-20T22:49:05.335Z 7 Acked 1006 Flush 0 Sent Done Done false
192152024-02-20T22:49:05.335Z 8 Acked 1007 Write 4 Sent Done Done false
192162024-02-20T22:49:05.335Z 9 Acked 1008 Read 1 Sent Done Done false
192172024-02-20T22:49:05.335Z 10 Acked 1009 Write 6 Sent Done Done false
192182024-02-20T22:49:05.335Z 11 Acked 1010 Write 9 Sent Done Done false
192192024-02-20T22:49:05.335Z 12 Acked 1011 Read 7 Sent Done Done false
192202024-02-20T22:49:05.335Z 13 Acked 1012 Read 1 Sent Done Done false
192212024-02-20T22:49:05.335Z 14 Acked 1013 Write 4 Sent Done Done false
192222024-02-20T22:49:05.335Z 15 Acked 1014 Write 7 Sent Done Done false
192232024-02-20T22:49:05.335Z 16 Acked 1015 Read 4 Sent Done Done false
192242024-02-20T22:49:05.335Z 17 Acked 1016 Flush 0 Sent Done Done false
192252024-02-20T22:49:05.335Z 18 Acked 1017 Write 4 Sent Done Done false
192262024-02-20T22:49:05.335Z 19 Acked 1018 Read 5 Sent Done Done false
192272024-02-20T22:49:05.335Z 20 Acked 1019 Write 3 Sent Done Done false
192282024-02-20T22:49:05.335Z 21 Acked 1020 Write 6 Sent Done Done false
192292024-02-20T22:49:05.335Z 22 Acked 1021 Read 6 Sent Done Done false
192302024-02-20T22:49:05.335Z 23 Acked 1022 Write 8 Sent Done Done false
192312024-02-20T22:49:05.335Z 24 Acked 1023 Read 10 Sent Done Done false
192322024-02-20T22:49:05.335Z 25 Acked 1024 Read 9 Sent Done Done false
192332024-02-20T22:49:05.335Z 26 Acked 1025 Flush 0 Sent Done Done false
192342024-02-20T22:49:05.335Z 27 Acked 1026 Write 3 Sent Done Done false
192352024-02-20T22:49:05.336Z 28 Acked 1027 Write 8 Sent Done Done false
192362024-02-20T22:49:05.336Z 29 Acked 1028 Read 6 Sent Done Done false
192372024-02-20T22:49:05.336Z 30 Acked 1029 Write 8 Sent Sent Sent false
192382024-02-20T22:49:05.336Z STATES DS:0 DS:1 DS:2 TOTAL
192392024-02-20T22:49:05.336Z New 0 0 0 0
192402024-02-20T22:49:05.336Z Sent 30 1 1 32
192412024-02-20T22:49:05.336Z Done 0 29 29 58
192422024-02-20T22:49:05.336Z Skipped 0 0 0 0
192432024-02-20T22:49:05.336Z Error 0 0 0 0
192442024-02-20T22:49:05.336ZLast Flush: 0 1025 1025
192452024-02-20T22:49:05.336ZDownstairs last five completed:
192462024-02-20T22:49:05.336ZUpstairs last five completed: 30 29 28 27 26
192472024-02-20T22:49:05.336Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:04.83578538Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1850}
192482024-02-20T22:49:05.336ZWrote out file "/tmp/test_repair_verify.data"
192492024-02-20T22:49:05.341Z
192502024-02-20T22:49:05.341ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192512024-02-20T22:49:05.353ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192522024-02-20T22:49:05.353Z 0 000-019 174 174 174 297 297 297 F T T <---
192532024-02-20T22:49:05.353Z 1 020-039 175 178 178 300 305 305 F T T <---
192542024-02-20T22:49:05.353Z 2 040-059 174 178 178 297 305 305 F T T <---
192552024-02-20T22:49:05.353Z 3 060-079 175 175 175 300 300 300 F F F
192562024-02-20T22:49:05.353Z 4 080-099 176 176 176 301 301 301 F F F
192572024-02-20T22:49:05.353Z 5 100-119 176 176 176 301 301 301 F F F
192582024-02-20T22:49:05.353Z 6 120-139 174 174 174 297 297 297 F F F
192592024-02-20T22:49:05.361Z 7 140-159 174 174 174 297 297 297 F F F
192602024-02-20T22:49:05.361Z 8 160-179 174 174 174 299 299 299 F T T <---
192612024-02-20T22:49:05.361Z 9 180-199 174 174 174 297 297 297 F F F
192622024-02-20T22:49:05.361Z 10 200-219 175 175 175 300 300 300 F F F
192632024-02-20T22:49:05.361Z 11 220-239 162 162 162 276 276 276 F F F
192642024-02-20T22:49:05.361Z 12 240-259 168 168 168 287 287 287 F F F
192652024-02-20T22:49:05.361Z 13 260-279 173 178 178 296 305 305 F F F <---
192662024-02-20T22:49:05.361Z 14 280-299 176 178 178 301 305 305 F F F <---
192672024-02-20T22:49:05.361Z 15 300-319 171 171 171 293 293 293 F F F
192682024-02-20T22:49:05.361Z 16 320-339 176 176 176 301 301 301 F F F
192692024-02-20T22:49:05.361Z 17 340-359 171 178 178 293 305 305 F F F <---
192702024-02-20T22:49:05.361Z 18 360-379 177 178 178 302 304 304 F F F <---
192712024-02-20T22:49:05.361Z 19 380-399 176 176 176 301 301 301 F F F
192722024-02-20T22:49:05.361Z 20 400-419 171 178 178 293 304 304 F F F <---
192732024-02-20T22:49:05.361Z 21 420-439 170 170 170 292 292 292 F F F
192742024-02-20T22:49:05.361Z 22 440-459 174 174 174 299 299 299 F F F
192752024-02-20T22:49:05.361Z 23 460-479 176 178 178 301 304 304 F F F <---
192762024-02-20T22:49:05.361Z 24 480-499 177 178 178 302 304 304 F F F <---
192772024-02-20T22:49:05.361Z 25 500-519 176 176 176 301 301 301 F F F
192782024-02-20T22:49:05.361Z 26 520-539 176 178 178 301 303 303 F F F <---
192792024-02-20T22:49:05.361Z 27 540-559 174 174 174 298 298 298 F F F
192802024-02-20T22:49:05.362Z 28 560-579 176 176 176 301 301 301 F F F
192812024-02-20T22:49:05.362Z 29 580-599 176 176 176 301 301 301 F F F
192822024-02-20T22:49:05.362ZMax gen: 178, Max flush: 305
192832024-02-20T22:49:05.362ZError: Difference in extent metadata found!
192842024-02-20T22:49:05.362ZA Difference in extent metadata is expected here
192852024-02-20T22:49:05.362ZOn loop 89
192862024-02-20T22:49:05.362Z
192872024-02-20T22:49:05.366ZVerifying data now
192882024-02-20T22:49:09.956ZNo change after verify
192892024-02-20T22:49:09.956ZLoop: 89 Downstairs dump after verify (and repair):
192902024-02-20T22:49:09.977ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192912024-02-20T22:49:09.977Z 0 000-019 179 179 179 306 306 306 F F F
192922024-02-20T22:49:09.977Z 1 020-039 179 179 179 306 306 306 F F F
192932024-02-20T22:49:09.977Z 2 040-059 179 179 179 306 306 306 F F F
192942024-02-20T22:49:09.977Z 3 060-079 175 175 175 300 300 300 F F F
192952024-02-20T22:49:09.977Z 4 080-099 176 176 176 301 301 301 F F F
192962024-02-20T22:49:09.985Z 5 100-119 176 176 176 301 301 301 F F F
192972024-02-20T22:49:09.985Z 6 120-139 174 174 174 297 297 297 F F F
192982024-02-20T22:49:09.985Z 7 140-159 174 174 174 297 297 297 F F F
192992024-02-20T22:49:09.985Z 8 160-179 179 179 179 306 306 306 F F F
193002024-02-20T22:49:09.985Z 9 180-199 174 174 174 297 297 297 F F F
193012024-02-20T22:49:09.985Z 10 200-219 175 175 175 300 300 300 F F F
193022024-02-20T22:49:09.985Z 11 220-239 162 162 162 276 276 276 F F F
193032024-02-20T22:49:09.985Z 12 240-259 168 168 168 287 287 287 F F F
193042024-02-20T22:49:09.985Z 13 260-279 178 178 178 305 305 305 F F F
193052024-02-20T22:49:09.985Z 14 280-299 178 178 178 305 305 305 F F F
193062024-02-20T22:49:09.985Z 15 300-319 171 171 171 293 293 293 F F F
193072024-02-20T22:49:09.985Z 16 320-339 176 176 176 301 301 301 F F F
193082024-02-20T22:49:09.985Z 17 340-359 178 178 178 305 305 305 F F F
193092024-02-20T22:49:09.986Z 18 360-379 178 178 178 304 304 304 F F F
193102024-02-20T22:49:09.986Z 19 380-399 176 176 176 301 301 301 F F F
193112024-02-20T22:49:09.986Z 20 400-419 178 178 178 304 304 304 F F F
193122024-02-20T22:49:09.986Z 21 420-439 170 170 170 292 292 292 F F F
193132024-02-20T22:49:09.986Z 22 440-459 174 174 174 299 299 299 F F F
193142024-02-20T22:49:09.986Z 23 460-479 178 178 178 304 304 304 F F F
193152024-02-20T22:49:09.986Z 24 480-499 178 178 178 304 304 304 F F F
193162024-02-20T22:49:09.986Z 25 500-519 176 176 176 301 301 301 F F F
193172024-02-20T22:49:09.986Z 26 520-539 178 178 178 303 303 303 F F F
193182024-02-20T22:49:09.986Z 27 540-559 174 174 174 298 298 298 F F F
193192024-02-20T22:49:09.986Z 28 560-579 176 176 176 301 301 301 F F F
193202024-02-20T22:49:09.986Z 29 580-599 176 176 176 301 301 301 F F F
193212024-02-20T22:49:09.986ZMax gen: 179, Max flush: 306
193222024-02-20T22:49:09.986Z
193232024-02-20T22:49:09.987ZBegin loop 90
193242024-02-20T22:49:09.987ZDownstairs to restart: 1
193252024-02-20T22:49:10.015Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493404417Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858}
193262024-02-20T22:49:10.015Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493664218Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858}
193272024-02-20T22:49:10.015Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493701767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858}
193282024-02-20T22:49:10.015Z{"msg":"Crucible eed93e9c-d8c9-48a2-b96e-3a1566458109 has session id: 321ea6a8-7057-4764-a65f-45532d55b8d1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493726006Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193292024-02-20T22:49:10.015Z{"msg":"Crucible runtime is spawned
193302024-02-20T22:49:10.015ZUpstairs opts: Upstairs UUID: eed93e9c-d8c9-48a2-b96e-3a1566458109, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493762075Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193312024-02-20T22:49:10.015Z{"msg":"Crucible stats registered with UUID: eed93e9c-d8c9-48a2-b96e-3a1566458109","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493788143Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193322024-02-20T22:49:10.015Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493816833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858}
193332024-02-20T22:49:10.015Z{"msg":"eed93e9c-d8c9-48a2-b96e-3a1566458109 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493947628Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193342024-02-20T22:49:10.015Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.493986467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193352024-02-20T22:49:10.015Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494026515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193362024-02-20T22:49:10.015Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494094013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193372024-02-20T22:49:10.015Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494130761Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193382024-02-20T22:49:10.015Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.4941612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193392024-02-20T22:49:10.015Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494203909Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193402024-02-20T22:49:10.015Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494232338Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193412024-02-20T22:49:10.015Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494264726Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193422024-02-20T22:49:10.016Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494291945Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193432024-02-20T22:49:10.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494326514Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193442024-02-20T22:49:10.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494361053Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193452024-02-20T22:49:10.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.494383462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193462024-02-20T22:49:10.016Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495051368Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193472024-02-20T22:49:10.016Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495091297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193482024-02-20T22:49:10.016Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495117246Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193492024-02-20T22:49:10.016Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495146815Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193502024-02-20T22:49:10.016Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495172424Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193512024-02-20T22:49:10.016Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495202063Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193522024-02-20T22:49:10.016Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495730774Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193532024-02-20T22:49:10.016Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.495800111Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193542024-02-20T22:49:10.016Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.49584251Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193552024-02-20T22:49:10.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496283494Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193562024-02-20T22:49:10.016Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496325843Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193572024-02-20T22:49:10.016Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496342022Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193582024-02-20T22:49:10.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.4964173Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193592024-02-20T22:49:10.016Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496445488Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193602024-02-20T22:49:10.016Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496460148Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193612024-02-20T22:49:10.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496473118Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193622024-02-20T22:49:10.016Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496487377Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193632024-02-20T22:49:10.016Z{"msg":"[0]R flush_numbers[0..12]: [306, 306, 306, 300, 301, 301, 297, 297, 306, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496509046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193642024-02-20T22:49:10.016Z{"msg":"[0]R generation[0..12]: [179, 179, 179, 175, 176, 176, 174, 174, 179, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496523736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193652024-02-20T22:49:10.016Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496539395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193662024-02-20T22:49:10.016Z{"msg":"[1]R flush_numbers[0..12]: [306, 306, 306, 300, 301, 301, 297, 297, 306, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496553685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193672024-02-20T22:49:10.017Z{"msg":"[1]R generation[0..12]: [179, 179, 179, 175, 176, 176, 174, 174, 179, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496574924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193682024-02-20T22:49:10.017Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496589493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193692024-02-20T22:49:10.017Z{"msg":"[2]R flush_numbers[0..12]: [306, 306, 306, 300, 301, 301, 297, 297, 306, 297, 300, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496604913Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193702024-02-20T22:49:10.017Z{"msg":"[2]R generation[0..12]: [179, 179, 179, 175, 176, 176, 174, 174, 179, 174, 175, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496628362Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193712024-02-20T22:49:10.017ZWait for a query_work_queue command to finish before sending IO
193722024-02-20T22:49:10.017Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.49666924Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193732024-02-20T22:49:10.017Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.49669359Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193742024-02-20T22:49:10.017Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496717799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193752024-02-20T22:49:10.017Z{"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496739418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193762024-02-20T22:49:10.017Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496776527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193772024-02-20T22:49:10.017Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496799586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193782024-02-20T22:49:10.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496820575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193792024-02-20T22:49:10.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496838634Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193802024-02-20T22:49:10.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496873183Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193812024-02-20T22:49:10.017Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496906482Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193822024-02-20T22:49:10.017Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496921062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193832024-02-20T22:49:10.017Z{"msg":"eed93e9c-d8c9-48a2-b96e-3a1566458109 is now active with session: 321ea6a8-7057-4764-a65f-45532d55b8d1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496934691Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193842024-02-20T22:49:10.017Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.49695292Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858,"session_id":"321ea6a8-7057-4764-a65f-45532d55b8d1"}
193852024-02-20T22:49:10.017Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:09.496975709Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858}
193862024-02-20T22:49:10.017ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193872024-02-20T22:49:10.034ZLoading write count information from file "/tmp/test_repair_verify.data"
193882024-02-20T22:49:10.034ZRun Repair workload
193892024-02-20T22:49:10.034Z01/30 Write block 18 len 28672 data: 15 13 14 10 14 11 10
193902024-02-20T22:49:10.034Z02/30 Write block 355 len 32768 data: 8 13 8 10 15 18 19 22
193912024-02-20T22:49:10.034Z03/30 Read block 556 len 24576
193922024-02-20T22:49:10.034Z04/30 Read block 344 len 24576
193932024-02-20T22:49:10.034Z05/30 Read block 3 len 4096
193942024-02-20T22:49:10.034Z06/30 Write block 547 len 8192 data: 14 12
193952024-02-20T22:49:10.034Z07/30 Write block 109 len 20480 data: 12 15 15 14 15
193962024-02-20T22:49:10.034Z08/30 Write block 392 len 8192 data: 16 15
193972024-02-20T22:49:10.034Z09/30 Read block 435 len 4096
193982024-02-20T22:49:10.034Z10/30 Write block 596 len 16384 data: 16 14 13 5
193992024-02-20T22:49:10.034Z11/30 Read block 342 len 28672
194002024-02-20T22:49:10.034Z12/30 Read block 428 len 24576
194012024-02-20T22:49:10.034Z13/30 Write block 409 len 20480 data: 15 16 16 18 22
194022024-02-20T22:49:10.034Z14/30 Read block 73 len 20480
194032024-02-20T22:49:10.035Z15/30 Read block 202 len 32768
194042024-02-20T22:49:10.035Z16/30 Write block 237 len 20480 data: 17 13 12 10 11
194052024-02-20T22:49:10.035Z17/30 Write block 136 len 32768 data: 10 10 10 11 13 12 14 14
194062024-02-20T22:49:10.035Z18/30 Write block 510 len 12288 data: 13 17 12
194072024-02-20T22:49:10.035Z19/30 Read block 340 len 36864
194082024-02-20T22:49:10.035Z20/30 Write block 380 len 32768 data: 14 11 14 13 14 12 10 9
194092024-02-20T22:49:10.035Z21/30 Write block 232 len 32768 data: 21 23 23 20 19 18 14 13
194102024-02-20T22:49:10.035Z22/30 Read block 185 len 4096
194112024-02-20T22:49:10.035Z23/30 Read block 195 len 24576
194122024-02-20T22:49:10.035Z24/30 Read block 223 len 40960
194132024-02-20T22:49:10.035Z25/30 Read block 233 len 36864
194142024-02-20T22:49:10.035Z26/30 Write block 461 len 36864 data: 22 21 22 18 18 13 17 20 22
194152024-02-20T22:49:10.035Z27/30 Read block 288 len 20480
194162024-02-20T22:49:10.035Z28/30 Write block 588 len 4096 data: 9
194172024-02-20T22:49:10.035Z29/30 Read block 497 len 24576
194182024-02-20T22:49:10.035Z30/30 Write block 50 len 8192 data: 12 12
194192024-02-20T22:49:10.035Z----------------------------------------------------------------
194202024-02-20T22:49:10.035Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
194212024-02-20T22:49:10.035ZStates: Active Active Active
194222024-02-20T22:49:10.035ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194232024-02-20T22:49:10.035Z 1 Acked 1000 Write 7 Done Sent Done false
194242024-02-20T22:49:10.035Z 2 Acked 1001 Write 8 Done Sent Done false
194252024-02-20T22:49:10.035Z 3 Acked 1002 Read 6 Done Sent Done false
194262024-02-20T22:49:10.036Z 4 Acked 1003 Read 6 Done Sent Done false
194272024-02-20T22:49:10.036Z 5 Acked 1004 Read 1 Done Sent Done false
194282024-02-20T22:49:10.036Z 6 Acked 1005 Write 2 Done Sent Done false
194292024-02-20T22:49:10.036Z 7 Acked 1006 Write 5 Done Sent Done false
194302024-02-20T22:49:10.036Z 8 Acked 1007 Write 2 Done Sent Done false
194312024-02-20T22:49:10.036Z 9 Acked 1008 Read 1 Done Sent Done false
194322024-02-20T22:49:10.036Z 10 Acked 1009 Write 4 Done Sent Done false
194332024-02-20T22:49:10.036Z 11 Acked 1010 Read 7 Done Sent Done false
194342024-02-20T22:49:10.036Z 12 Acked 1011 Read 6 Done Sent Done false
194352024-02-20T22:49:10.036Z 13 Acked 1012 Write 5 Done Sent Done false
194362024-02-20T22:49:10.036Z 14 Acked 1013 Read 5 Done Sent Done false
194372024-02-20T22:49:10.036Z 15 Acked 1014 Read 8 Done Sent Done false
194382024-02-20T22:49:10.036Z 16 Acked 1015 Write 5 Done Sent Done false
194392024-02-20T22:49:10.036Z 17 Acked 1016 Write 8 Done Sent Done false
194402024-02-20T22:49:10.036Z 18 Acked 1017 Write 3 Done Sent Done false
194412024-02-20T22:49:10.036Z 19 Acked 1018 Read 9 Done Sent Done false
194422024-02-20T22:49:10.036Z 20 Acked 1019 Write 8 Done Sent Done false
194432024-02-20T22:49:10.036Z 21 Acked 1020 Write 8 Done Sent Done false
194442024-02-20T22:49:10.036Z 22 Acked 1021 Read 1 Done Sent Done false
194452024-02-20T22:49:10.036Z 23 Acked 1022 Read 6 Done Sent Done false
194462024-02-20T22:49:10.036Z 24 Acked 1023 Read 10 Done Sent Done false
194472024-02-20T22:49:10.036Z 25 Acked 1024 Read 9 Done Sent Done false
194482024-02-20T22:49:10.036Z 26 Acked 1025 Write 9 Done Sent Done false
194492024-02-20T22:49:10.036Z 27 Acked 1026 Read 5 Done Sent Done false
194502024-02-20T22:49:10.036Z 28 Acked 1027 Write 1 Done Sent Done false
194512024-02-20T22:49:10.036Z 29 Acked 1028 Read 6 Done Sent Done false
194522024-02-20T22:49:10.037Z 30 Acked 1029 Write 2 Sent Sent Sent false
194532024-02-20T22:49:10.037Z STATES DS:0 DS:1 DS:2 TOTAL
194542024-02-20T22:49:10.037Z New 0 0 0 0
194552024-02-20T22:49:10.037Z Sent 1 30 1 32
194562024-02-20T22:49:10.037Z Done 29 0 29 58
194572024-02-20T22:49:10.037Z Skipped 0 0 0 0
194582024-02-20T22:49:10.037Z Error 0 0 0 0
194592024-02-20T22:49:10.037ZLast Flush: 0 0 0
194602024-02-20T22:49:10.037ZDownstairs last five completed:
194612024-02-20T22:49:10.037ZUpstairs last five completed: 30 29 28 27 26
194622024-02-20T22:49:10.037Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:09.5237996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1858}
194632024-02-20T22:49:10.037ZWrote out file "/tmp/test_repair_verify.data"
194642024-02-20T22:49:10.037Z
194652024-02-20T22:49:10.042ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194662024-02-20T22:49:10.055ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194672024-02-20T22:49:10.055Z 0 000-019 179 179 179 306 306 306 T F T <---
194682024-02-20T22:49:10.055Z 1 020-039 179 179 179 306 306 306 T F T <---
194692024-02-20T22:49:10.055Z 2 040-059 179 179 179 306 306 306 T F T <---
194702024-02-20T22:49:10.056Z 3 060-079 175 175 175 300 300 300 F F F
194712024-02-20T22:49:10.056Z 4 080-099 176 176 176 301 301 301 F F F
194722024-02-20T22:49:10.056Z 5 100-119 176 176 176 301 301 301 T F T <---
194732024-02-20T22:49:10.056Z 6 120-139 174 174 174 297 297 297 T F T <---
194742024-02-20T22:49:10.056Z 7 140-159 174 174 174 297 297 297 T F T <---
194752024-02-20T22:49:10.056Z 8 160-179 179 179 179 306 306 306 F F F
194762024-02-20T22:49:10.056Z 9 180-199 174 174 174 297 297 297 F F F
194772024-02-20T22:49:10.056Z 10 200-219 175 175 175 300 300 300 F F F
194782024-02-20T22:49:10.056Z 11 220-239 162 162 162 276 276 276 T F T <---
194792024-02-20T22:49:10.056Z 12 240-259 168 168 168 287 287 287 T F T <---
194802024-02-20T22:49:10.056Z 13 260-279 178 178 178 305 305 305 F F F
194812024-02-20T22:49:10.056Z 14 280-299 178 178 178 305 305 305 F F F
194822024-02-20T22:49:10.056Z 15 300-319 171 171 171 293 293 293 F F F
194832024-02-20T22:49:10.057Z 16 320-339 176 176 176 301 301 301 F F F
194842024-02-20T22:49:10.057Z 17 340-359 178 178 178 305 305 305 T F T <---
194852024-02-20T22:49:10.057Z 18 360-379 178 178 178 304 304 304 T F T <---
194862024-02-20T22:49:10.057Z 19 380-399 176 176 176 301 301 301 T F T <---
194872024-02-20T22:49:10.057Z 20 400-419 178 178 178 304 304 304 T F T <---
194882024-02-20T22:49:10.057Z 21 420-439 170 170 170 292 292 292 F F F
194892024-02-20T22:49:10.057Z 22 440-459 174 174 174 299 299 299 F F F
194902024-02-20T22:49:10.057Z 23 460-479 178 178 178 304 304 304 T F T <---
194912024-02-20T22:49:10.057Z 24 480-499 178 178 178 304 304 304 F F F
194922024-02-20T22:49:10.057Z 25 500-519 176 176 176 301 301 301 T F T <---
194932024-02-20T22:49:10.057Z 26 520-539 178 178 178 303 303 303 F F F
194942024-02-20T22:49:10.057Z 27 540-559 174 174 174 298 298 298 T F T <---
194952024-02-20T22:49:10.057Z 28 560-579 176 176 176 301 301 301 F F F
194962024-02-20T22:49:10.058Z 29 580-599 176 176 176 301 301 301 T F T <---
194972024-02-20T22:49:10.058ZMax gen: 179, Max flush: 306
194982024-02-20T22:49:10.058ZError: Difference in extent metadata found!
194992024-02-20T22:49:10.058ZA Difference in extent metadata is expected here
195002024-02-20T22:49:10.058ZOn loop 90
195012024-02-20T22:49:10.058Z
195022024-02-20T22:49:10.064ZVerifying data now
195032024-02-20T22:49:14.842ZNo change after verify
195042024-02-20T22:49:14.842ZLoop: 90 Downstairs dump after verify (and repair):
195052024-02-20T22:49:14.858ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195062024-02-20T22:49:14.858Z 0 000-019 180 180 180 307 307 307 F F F
195072024-02-20T22:49:14.858Z 1 020-039 180 180 180 307 307 307 F F F
195082024-02-20T22:49:14.858Z 2 040-059 180 180 180 307 307 307 F F F
195092024-02-20T22:49:14.858Z 3 060-079 175 175 175 300 300 300 F F F
195102024-02-20T22:49:14.858Z 4 080-099 176 176 176 301 301 301 F F F
195112024-02-20T22:49:14.858Z 5 100-119 180 180 180 307 307 307 F F F
195122024-02-20T22:49:14.858Z 6 120-139 180 180 180 307 307 307 F F F
195132024-02-20T22:49:14.858Z 7 140-159 180 180 180 307 307 307 F F F
195142024-02-20T22:49:14.858Z 8 160-179 179 179 179 306 306 306 F F F
195152024-02-20T22:49:14.858Z 9 180-199 174 174 174 297 297 297 F F F
195162024-02-20T22:49:14.864Z 10 200-219 175 175 175 300 300 300 F F F
195172024-02-20T22:49:14.865Z 11 220-239 180 180 180 307 307 307 F F F
195182024-02-20T22:49:14.865Z 12 240-259 180 180 180 307 307 307 F F F
195192024-02-20T22:49:14.865Z 13 260-279 178 178 178 305 305 305 F F F
195202024-02-20T22:49:14.865Z 14 280-299 178 178 178 305 305 305 F F F
195212024-02-20T22:49:14.865Z 15 300-319 171 171 171 293 293 293 F F F
195222024-02-20T22:49:14.865Z 16 320-339 176 176 176 301 301 301 F F F
195232024-02-20T22:49:14.865Z 17 340-359 180 180 180 307 307 307 F F F
195242024-02-20T22:49:14.865Z 18 360-379 180 180 180 307 307 307 F F F
195252024-02-20T22:49:14.865Z 19 380-399 180 180 180 307 307 307 F F F
195262024-02-20T22:49:14.865Z 20 400-419 180 180 180 307 307 307 F F F
195272024-02-20T22:49:14.865Z 21 420-439 170 170 170 292 292 292 F F F
195282024-02-20T22:49:14.865Z 22 440-459 174 174 174 299 299 299 F F F
195292024-02-20T22:49:14.865Z 23 460-479 180 180 180 307 307 307 F F F
195302024-02-20T22:49:14.865Z 24 480-499 178 178 178 304 304 304 F F F
195312024-02-20T22:49:14.865Z 25 500-519 180 180 180 307 307 307 F F F
195322024-02-20T22:49:14.865Z 26 520-539 178 178 178 303 303 303 F F F
195332024-02-20T22:49:14.865Z 27 540-559 180 180 180 307 307 307 F F F
195342024-02-20T22:49:14.865Z 28 560-579 176 176 176 301 301 301 F F F
195352024-02-20T22:49:14.865Z 29 580-599 180 180 180 307 307 307 F F F
195362024-02-20T22:49:14.865ZMax gen: 180, Max flush: 307
195372024-02-20T22:49:14.865Z
195382024-02-20T22:49:14.865ZBegin loop 91
195392024-02-20T22:49:14.865ZDownstairs to restart: 1
195402024-02-20T22:49:14.878Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381380452Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866}
195412024-02-20T22:49:14.878Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381630123Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866}
195422024-02-20T22:49:14.878Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381647533Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866}
195432024-02-20T22:49:14.878Z{"msg":"Crucible 52d090cf-6c45-4073-a2dc-310f70927b5c has session id: fbe814a2-9bbd-4b41-995f-73149c7a0497","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381661612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195442024-02-20T22:49:14.878Z{"msg":"Upstairs opts: Upstairs UUID: 52d090cf-6c45-4073-a2dc-310f70927b5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381682432Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195452024-02-20T22:49:14.878Z{"msg":"Crucible stats registered with UUID: 52d090cf-6c45-4073-a2dc-310f70927b5c","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381704621Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195462024-02-20T22:49:14.878ZCrucible runtime is spawned
195472024-02-20T22:49:14.891Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.381842306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866}
195482024-02-20T22:49:14.891Z{"msg":"52d090cf-6c45-4073-a2dc-310f70927b5c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.38201836Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195492024-02-20T22:49:14.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382042879Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195502024-02-20T22:49:14.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382067428Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195512024-02-20T22:49:14.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382094727Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195522024-02-20T22:49:14.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382143265Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195532024-02-20T22:49:14.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382173374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195542024-02-20T22:49:14.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382217933Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195552024-02-20T22:49:14.892Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382247401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195562024-02-20T22:49:14.892Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.38228794Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195572024-02-20T22:49:14.892Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382340998Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195582024-02-20T22:49:14.892Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382420005Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195592024-02-20T22:49:14.892Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382453364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195602024-02-20T22:49:14.892Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.382547401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195612024-02-20T22:49:14.892Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383090861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195622024-02-20T22:49:14.892Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.38312811Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195632024-02-20T22:49:14.892Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383144889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195642024-02-20T22:49:14.892Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383167219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195652024-02-20T22:49:14.892Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383213067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195662024-02-20T22:49:14.892Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383256195Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195672024-02-20T22:49:14.892Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383791576Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195682024-02-20T22:49:14.892Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383827915Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195692024-02-20T22:49:14.892Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.383937471Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195702024-02-20T22:49:14.892Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384366225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195712024-02-20T22:49:14.893Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384409254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195722024-02-20T22:49:14.893Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384471612Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195732024-02-20T22:49:14.893Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.38451321Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195742024-02-20T22:49:14.893Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.38452995Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195752024-02-20T22:49:14.893Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384551489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195762024-02-20T22:49:14.893Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384564789Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195772024-02-20T22:49:14.893Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384584918Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195782024-02-20T22:49:14.893Z{"msg":"[0]R flush_numbers[0..12]: [307, 307, 307, 300, 301, 307, 307, 307, 306, 297, 300, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384646026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195792024-02-20T22:49:14.893Z{"msg":"[0]R generation[0..12]: [180, 180, 180, 175, 176, 180, 180, 180, 179, 174, 175, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384674545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195802024-02-20T22:49:14.893Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384689914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195812024-02-20T22:49:14.893Z{"msg":"[1]R flush_numbers[0..12]: [307, 307, 307, 300, 301, 307, 307, 307, 306, 297, 300, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384710923Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195822024-02-20T22:49:14.893Z{"msg":"[1]R generation[0..12]: [180, 180, 180, 175, 176, 180, 180, 180, 179, 174, 175, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384725193Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195832024-02-20T22:49:14.893Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384774631Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195842024-02-20T22:49:14.893Z{"msg":"[2]R flush_numbers[0..12]: [307, 307, 307, 300, 301, 307, 307, 307, 306, 297, 300, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.38478911Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
195852024-02-20T22:49:14.893Zdownstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195862024-02-20T22:49:14.893Z{"msg":"[2]R generation[0..12]: [180, 180, 180, 175, 176, 180, 180, 180, 179, 174, 175, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.3848086Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195872024-02-20T22:49:14.893Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384834999Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195882024-02-20T22:49:14.893Z{"msg":"Max found gen is 181","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384851768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195892024-02-20T22:49:14.893Z{"msg":"Generation requested: 182 >= found:181","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384874267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195902024-02-20T22:49:14.893Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384887947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195912024-02-20T22:49:14.893Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384902257Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195922024-02-20T22:49:14.893Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384917046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195932024-02-20T22:49:14.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384929236Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195942024-02-20T22:49:14.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384950575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195952024-02-20T22:49:14.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384964614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195962024-02-20T22:49:14.894Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.384990333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195972024-02-20T22:49:14.894Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.385004393Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195982024-02-20T22:49:14.894Z{"msg":"52d090cf-6c45-4073-a2dc-310f70927b5c is now active with session: fbe814a2-9bbd-4b41-995f-73149c7a0497","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.385017602Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
195992024-02-20T22:49:14.894Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.385029582Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866,"session_id":"fbe814a2-9bbd-4b41-995f-73149c7a0497"}
196002024-02-20T22:49:14.894Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:14.385041932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866}
196012024-02-20T22:49:14.894ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196022024-02-20T22:49:14.894ZLoading write count information from file "/tmp/test_repair_verify.data"
196032024-02-20T22:49:14.894ZRun Repair workload
196042024-02-20T22:49:14.894Z01/30 Write block 467 len 8192 data: 18 21
196052024-02-20T22:49:14.894Z02/30 Flush
196062024-02-20T22:49:14.904Z03/30 Write block 133 len 32768 data: 10 8 13 11 11 11 12 14
196072024-02-20T22:49:14.904Z04/30 Write block 409 len 36864 data: 16 17 17 19 23 23 22 18 16
196082024-02-20T22:49:14.904Z05/30 Write block 54 len 12288 data: 14 12 12
196092024-02-20T22:49:14.904Z06/30 Read block 278 len 24576
196102024-02-20T22:49:14.904Z07/30 Write block 407 len 24576 data: 15 13 17 18 18 20
196112024-02-20T22:49:14.904Z08/30 Flush
196122024-02-20T22:49:14.904Z09/30 Write block 330 len 40960 data: 13 12 17 18 16 13 15 15 17 16
196132024-02-20T22:49:14.904Z10/30 Read block 57 len 40960
196142024-02-20T22:49:14.904Z11/30 Read block 504 len 32768
196152024-02-20T22:49:14.904Z12/30 Write block 318 len 20480 data: 14 14 16 16 19
196162024-02-20T22:49:14.904Z13/30 Read block 583 len 32768
196172024-02-20T22:49:14.904Z14/30 Flush
196182024-02-20T22:49:14.904Z15/30 Write block 213 len 28672 data: 10 10 11 10 7 7 7
196192024-02-20T22:49:14.904Z16/30 Read block 378 len 36864
196202024-02-20T22:49:14.904Z17/30 Write block 156 len 4096 data: 11
196212024-02-20T22:49:14.904Z18/30 Write block 88 len 16384 data: 10 10 12 12
196222024-02-20T22:49:14.910Z19/30 Write block 186 len 12288 data: 14 16 14
196232024-02-20T22:49:14.910Z20/30 Read block 153 len 8192
196242024-02-20T22:49:14.910Z21/30 Flush
196252024-02-20T22:49:14.910Z22/30 Write block 436 len 20480 data: 7 7 10 13 16
196262024-02-20T22:49:14.910Z23/30 Flush
196272024-02-20T22:49:14.917Z24/30 Write block 481 len 8192 data: 20 20
196282024-02-20T22:49:14.917Z25/30 Read block 305 len 28672
196292024-02-20T22:49:14.917Z26/30 Write block 9 len 4096 data: 9
196302024-02-20T22:49:14.917Z27/30 Write block 121 len 20480 data: 17 19 17 16 12
196312024-02-20T22:49:14.917Z28/30 Read block 172 len 12288
196322024-02-20T22:49:14.917Z29/30 Read block 114 len 36864
196332024-02-20T22:49:14.918Z30/30 Write block 161 len 20480 data: 10 11 10 10 13
196342024-02-20T22:49:14.918Z----------------------------------------------------------------
196352024-02-20T22:49:14.918Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196362024-02-20T22:49:14.918ZStates: Active Active Active
196372024-02-20T22:49:14.918ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196382024-02-20T22:49:14.918Z 1 Acked 1000 Write 2 Done Done Done false
196392024-02-20T22:49:14.918Z 2 Acked 1001 Flush 0 Done Sent Done false
196402024-02-20T22:49:14.918Z 3 Acked 1002 Write 8 Done Sent Done false
196412024-02-20T22:49:14.918Z 4 Acked 1003 Write 9 Done Sent Done false
196422024-02-20T22:49:14.918Z 5 Acked 1004 Write 3 Done Sent Done false
196432024-02-20T22:49:14.918Z 6 Acked 1005 Read 6 Done Sent Done false
196442024-02-20T22:49:14.918Z 7 Acked 1006 Write 6 Done Sent Done false
196452024-02-20T22:49:14.918Z 8 Acked 1007 Flush 0 Done Sent Done false
196462024-02-20T22:49:14.918Z 9 Acked 1008 Write 10 Done Sent Done false
196472024-02-20T22:49:14.918Z 10 Acked 1009 Read 10 Done Sent Done false
196482024-02-20T22:49:14.918Z 11 Acked 1010 Read 8 Done Sent Done false
196492024-02-20T22:49:14.918Z 12 Acked 1011 Write 5 Done Sent Done false
196502024-02-20T22:49:14.918Z 13 Acked 1012 Read 8 Done Sent Done false
196512024-02-20T22:49:14.918Z 14 Acked 1013 Flush 0 Done Sent Done false
196522024-02-20T22:49:14.918Z 15 Acked 1014 Write 7 Done Sent Done false
196532024-02-20T22:49:14.918Z 16 Acked 1015 Read 9 Done Sent Done false
196542024-02-20T22:49:14.918Z 17 Acked 1016 Write 1 Done Sent Done false
196552024-02-20T22:49:14.918Z 18 Acked 1017 Write 4 Done Sent Done false
196562024-02-20T22:49:14.918Z 19 Acked 1018 Write 3 Done Sent Done false
196572024-02-20T22:49:14.919Z 20 Acked 1019 Read 2 Done Sent Done false
196582024-02-20T22:49:14.919Z 21 Acked 1020 Flush 0 Done Sent Done false
196592024-02-20T22:49:14.919Z 22 Acked 1021 Write 5 Done Sent Done false
196602024-02-20T22:49:14.919Z 23 Acked 1022 Flush 0 Done Sent Done false
196612024-02-20T22:49:14.919Z 24 Acked 1023 Write 2 Done Sent Done false
196622024-02-20T22:49:14.919Z 25 Acked 1024 Read 7 Done Sent Done false
196632024-02-20T22:49:14.919Z 26 Acked 1025 Write 1 Done Sent Done false
196642024-02-20T22:49:14.919Z 27 Acked 1026 Write 5 Done Sent Done false
196652024-02-20T22:49:14.919Z 28 Acked 1027 Read 3 Done Sent Done false
196662024-02-20T22:49:14.919Z 29 Acked 1028 Read 9 Done Sent Done false
196672024-02-20T22:49:14.919Z 30 Acked 1029 Write 5 Sent Sent Sent false
196682024-02-20T22:49:14.919Z STATES DS:0 DS:1 DS:2 TOTAL
196692024-02-20T22:49:14.919Z New 0 0 0 0
196702024-02-20T22:49:14.919Z Sent 1 29 1 31
196712024-02-20T22:49:14.919Z Done 29 1 29 59
196722024-02-20T22:49:14.919Z Skipped 0 0 0 0
196732024-02-20T22:49:14.919Z Error 0 0 0 0
196742024-02-20T22:49:14.919ZLast Flush: 1022 0 1022
196752024-02-20T22:49:14.919ZDownstairs last five completed:
196762024-02-20T22:49:14.919ZUpstairs last five completed: 30 29 28 27 26
196772024-02-20T22:49:14.919Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:14.419903264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1866}
196782024-02-20T22:49:14.919ZWrote out file "/tmp/test_repair_verify.data"
196792024-02-20T22:49:14.929Z
196802024-02-20T22:49:14.929ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196812024-02-20T22:49:14.938ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196822024-02-20T22:49:14.938Z 0 000-019 180 180 180 307 307 307 T F T <---
196832024-02-20T22:49:14.938Z 1 020-039 180 180 180 307 307 307 F F F
196842024-02-20T22:49:14.938Z 2 040-059 182 180 182 309 307 309 F F F <---
196852024-02-20T22:49:14.938Z 3 060-079 175 175 175 300 300 300 F F F
196862024-02-20T22:49:14.938Z 4 080-099 182 176 182 311 301 311 F F F <---
196872024-02-20T22:49:14.938Z 5 100-119 180 180 180 307 307 307 F F F
196882024-02-20T22:49:14.938Z 6 120-139 182 180 182 309 307 309 T F T <---
196892024-02-20T22:49:14.938Z 7 140-159 182 180 182 311 307 311 F F F <---
196902024-02-20T22:49:14.939Z 8 160-179 179 179 179 306 306 306 T F T <---
196912024-02-20T22:49:14.939Z 9 180-199 182 174 182 311 297 311 F F F <---
196922024-02-20T22:49:14.951Z 10 200-219 182 175 182 311 300 311 F F F <---
196932024-02-20T22:49:14.951Z 11 220-239 180 180 180 307 307 307 F F F
196942024-02-20T22:49:14.951Z 12 240-259 180 180 180 307 307 307 F F F
196952024-02-20T22:49:14.951Z 13 260-279 178 178 178 305 305 305 F F F
196962024-02-20T22:49:14.951Z 14 280-299 178 178 178 305 305 305 F F F
196972024-02-20T22:49:14.951Z 15 300-319 182 171 182 310 293 310 F F F <---
196982024-02-20T22:49:14.951Z 16 320-339 182 176 182 310 301 310 F F F <---
196992024-02-20T22:49:14.951Z 17 340-359 180 180 180 307 307 307 F F F
197002024-02-20T22:49:14.951Z 18 360-379 180 180 180 307 307 307 F F F
197012024-02-20T22:49:14.951Z 19 380-399 180 180 180 307 307 307 F F F
197022024-02-20T22:49:14.951Z 20 400-419 182 180 182 309 307 309 F F F <---
197032024-02-20T22:49:14.951Z 21 420-439 182 170 182 312 292 312 F F F <---
197042024-02-20T22:49:14.951Z 22 440-459 182 174 182 312 299 312 F F F <---
197052024-02-20T22:49:14.952Z 23 460-479 182 180 182 308 307 308 F T F <---
197062024-02-20T22:49:14.952Z 24 480-499 178 178 178 304 304 304 T F T <---
197072024-02-20T22:49:14.952Z 25 500-519 180 180 180 307 307 307 F F F
197082024-02-20T22:49:14.952Z 26 520-539 178 178 178 303 303 303 F F F
197092024-02-20T22:49:14.952Z 27 540-559 180 180 180 307 307 307 F F F
197102024-02-20T22:49:14.952Z 28 560-579 176 176 176 301 301 301 F F F
197112024-02-20T22:49:14.952Z 29 580-599 180 180 180 307 307 307 F F F
197122024-02-20T22:49:14.952ZMax gen: 182, Max flush: 312
197132024-02-20T22:49:14.952ZError: Difference in extent metadata found!
197142024-02-20T22:49:14.952ZA Difference in extent metadata is expected here
197152024-02-20T22:49:14.952ZOn loop 91
197162024-02-20T22:49:14.952Z
197172024-02-20T22:49:14.952ZVerifying data now
197182024-02-20T22:49:19.664ZNo change after verify
197192024-02-20T22:49:19.664ZLoop: 91 Downstairs dump after verify (and repair):
197202024-02-20T22:49:19.677ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197212024-02-20T22:49:19.677Z 0 000-019 183 183 183 313 313 313 F F F
197222024-02-20T22:49:19.678Z 1 020-039 180 180 180 307 307 307 F F F
197232024-02-20T22:49:19.678Z 2 040-059 182 182 182 309 309 309 F F F
197242024-02-20T22:49:19.678Z 3 060-079 175 175 175 300 300 300 F F F
197252024-02-20T22:49:19.682Z 4 080-099 182 182 182 311 311 311 F F F
197262024-02-20T22:49:19.682Z 5 100-119 180 180 180 307 307 307 F F F
197272024-02-20T22:49:19.682Z 6 120-139 183 183 183 313 313 313 F F F
197282024-02-20T22:49:19.682Z 7 140-159 182 182 182 311 311 311 F F F
197292024-02-20T22:49:19.682Z 8 160-179 183 183 183 313 313 313 F F F
197302024-02-20T22:49:19.682Z 9 180-199 182 182 182 311 311 311 F F F
197312024-02-20T22:49:19.682Z 10 200-219 182 182 182 311 311 311 F F F
197322024-02-20T22:49:19.682Z 11 220-239 180 180 180 307 307 307 F F F
197332024-02-20T22:49:19.682Z 12 240-259 180 180 180 307 307 307 F F F
197342024-02-20T22:49:19.682Z 13 260-279 178 178 178 305 305 305 F F F
197352024-02-20T22:49:19.682Z 14 280-299 178 178 178 305 305 305 F F F
197362024-02-20T22:49:19.682Z 15 300-319 182 182 182 310 310 310 F F F
197372024-02-20T22:49:19.682Z 16 320-339 182 182 182 310 310 310 F F F
197382024-02-20T22:49:19.682Z 17 340-359 180 180 180 307 307 307 F F F
197392024-02-20T22:49:19.682Z 18 360-379 180 180 180 307 307 307 F F F
197402024-02-20T22:49:19.683Z 19 380-399 180 180 180 307 307 307 F F F
197412024-02-20T22:49:19.683Z 20 400-419 182 182 182 309 309 309 F F F
197422024-02-20T22:49:19.683Z 21 420-439 182 182 182 312 312 312 F F F
197432024-02-20T22:49:19.683Z 22 440-459 182 182 182 312 312 312 F F F
197442024-02-20T22:49:19.683Z 23 460-479 182 182 182 308 308 308 F F F
197452024-02-20T22:49:19.683Z 24 480-499 183 183 183 313 313 313 F F F
197462024-02-20T22:49:19.683Z 25 500-519 180 180 180 307 307 307 F F F
197472024-02-20T22:49:19.683Z 26 520-539 178 178 178 303 303 303 F F F
197482024-02-20T22:49:19.683Z 27 540-559 180 180 180 307 307 307 F F F
197492024-02-20T22:49:19.683Z 28 560-579 176 176 176 301 301 301 F F F
197502024-02-20T22:49:19.683Z 29 580-599 180 180 180 307 307 307 F F F
197512024-02-20T22:49:19.683ZMax gen: 183, Max flush: 313
197522024-02-20T22:49:19.683Z
197532024-02-20T22:49:19.683ZBegin loop 92
197542024-02-20T22:49:19.683ZDownstairs to restart: 1
197552024-02-20T22:49:19.699Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202104023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874}
197562024-02-20T22:49:19.699Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202370403Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874}
197572024-02-20T22:49:19.699Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202388333Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874}
197582024-02-20T22:49:19.709Z{"msg":"Crucible 71e5c776-f040-4b3e-b881-89bdbc80b30f has session id: c3e9343d-f40f-4283-ab2e-168c85881574","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202420521Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197592024-02-20T22:49:19.709ZCrucible runtime is spawned
197602024-02-20T22:49:19.709Z{"msg":"Upstairs opts: Upstairs UUID: 71e5c776-f040-4b3e-b881-89bdbc80b30f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202473159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197612024-02-20T22:49:19.709Z{"msg":"Crucible stats registered with UUID: 71e5c776-f040-4b3e-b881-89bdbc80b30f","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202518418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197622024-02-20T22:49:19.709Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202545217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874}
197632024-02-20T22:49:19.709Z{"msg":"71e5c776-f040-4b3e-b881-89bdbc80b30f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202682922Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197642024-02-20T22:49:19.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202706771Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197652024-02-20T22:49:19.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202721931Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197662024-02-20T22:49:19.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.20273588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197672024-02-20T22:49:19.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.20274997Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197682024-02-20T22:49:19.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202782768Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197692024-02-20T22:49:19.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202809458Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197702024-02-20T22:49:19.709Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202848586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197712024-02-20T22:49:19.709Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202895934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197722024-02-20T22:49:19.709Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.202923793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197732024-02-20T22:49:19.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203060929Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197742024-02-20T22:49:19.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203098217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197752024-02-20T22:49:19.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203124976Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197762024-02-20T22:49:19.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.2038568Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197772024-02-20T22:49:19.710Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203895889Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197782024-02-20T22:49:19.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203921888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197792024-02-20T22:49:19.710Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203950527Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197802024-02-20T22:49:19.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.203974636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197812024-02-20T22:49:19.710Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.204029494Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197822024-02-20T22:49:19.710Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.204526946Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197832024-02-20T22:49:19.710Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.204568804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197842024-02-20T22:49:19.710Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.204601073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197852024-02-20T22:49:19.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205105385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197862024-02-20T22:49:19.710Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205144384Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197872024-02-20T22:49:19.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205171833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197882024-02-20T22:49:19.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205217801Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197892024-02-20T22:49:19.710Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.20524467Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197902024-02-20T22:49:19.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205273699Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197912024-02-20T22:49:19.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205296209Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197922024-02-20T22:49:19.710Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205324217Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197932024-02-20T22:49:19.710Z{"msg":"[0]R flush_numbers[0..12]: [313, 307, 309, 300, 311, 307, 313, 311, 313, 311, 311, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205349716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197942024-02-20T22:49:19.710Z{"msg":"[0]R generation[0..12]: [183, 180, 182, 175, 182, 180, 183, 182, 183, 182, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205378206Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197952024-02-20T22:49:19.710Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205403965Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197962024-02-20T22:49:19.710Z{"msg":"[1]R flush_numbers[0..12]: [313, 307, 309, 300, 311, 307, 313, 311, 313, 311, 311, 307]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
197972024-02-20T22:49:19.710Z,"time":"2024-02-20T22:49:19.205437933Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197982024-02-20T22:49:19.710Z{"msg":"[1]R generation[0..12]: [183, 180, 182, 175, 182, 180, 183, 182, 183, 182, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205474132Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
197992024-02-20T22:49:19.710Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205508861Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198002024-02-20T22:49:19.711Z{"msg":"[2]R flush_numbers[0..12]: [313, 307, 309, 300, 311, 307, 313, 311, 313, 311, 311, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.20554473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198012024-02-20T22:49:19.711Z{"msg":"[2]R generation[0..12]: [183, 180, 182, 175, 182, 180, 183, 182, 183, 182, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205578548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198022024-02-20T22:49:19.711Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205606138Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198032024-02-20T22:49:19.711Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205639086Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198042024-02-20T22:49:19.711Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205663935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198052024-02-20T22:49:19.711Z{"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205702934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198062024-02-20T22:49:19.711Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205735873Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198072024-02-20T22:49:19.711Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205769381Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198082024-02-20T22:49:19.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205801Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198092024-02-20T22:49:19.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.20582109Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198102024-02-20T22:49:19.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205850569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198112024-02-20T22:49:19.711Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205867558Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198122024-02-20T22:49:19.711Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198132024-02-20T22:49:19.711Z,"time":"2024-02-20T22:49:19.205895907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198142024-02-20T22:49:19.711Z{"msg":"71e5c776-f040-4b3e-b881-89bdbc80b30f is now active with session: c3e9343d-f40f-4283-ab2e-168c85881574","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205913587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198152024-02-20T22:49:19.711Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205942745Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874,"session_id":"c3e9343d-f40f-4283-ab2e-168c85881574"}
198162024-02-20T22:49:19.711Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:19.205957305Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874}
198172024-02-20T22:49:19.711ZLoading write count information from file "/tmp/test_repair_verify.data"
198182024-02-20T22:49:19.711ZRun Repair workload
198192024-02-20T22:49:19.711Z01/30 Write block 179 len 36864 data: 18 17 21 19 15 17 14 15 17
198202024-02-20T22:49:19.716Z02/30 Flush
198212024-02-20T22:49:19.716Z03/30 Write block 466 len 32768 data: 14 19 22 23 17 14 13 12
198222024-02-20T22:49:19.716Z04/30 Flush
198232024-02-20T22:49:19.716Z05/30 Write block 488 len 36864 data: 17 14 12 13 16 18 18 15 16
198242024-02-20T22:49:19.716Z06/30 Write block 528 len 4096 data: 14
198252024-02-20T22:49:19.716Z07/30 Write block 21 len 36864 data: 11 15 12 11 16 16 21 21 25
198262024-02-20T22:49:19.716Z08/30 Read block 371 len 40960
198272024-02-20T22:49:19.716Z09/30 Read block 244 len 36864
198282024-02-20T22:49:19.716Z10/30 Read block 463 len 24576
198292024-02-20T22:49:19.720Z11/30 Flush
198302024-02-20T22:49:19.720Z12/30 Write block 386 len 16384 data: 11 10 10 15
198312024-02-20T22:49:19.720Z13/30 Write block 519 len 20480 data: 13 15 13 12 11
198322024-02-20T22:49:19.720Z14/30 Read block 418 len 16384
198332024-02-20T22:49:19.724Z15/30 Write block 388 len 8192 data: 11 16
198342024-02-20T22:49:19.724Z16/30 Write block 534 len 16384 data: 17 20 18 16
198352024-02-20T22:49:19.724Z17/30 Read block 471 len 12288
198362024-02-20T22:49:19.724Z18/30 Read block 358 len 8192
198372024-02-20T22:49:19.724Z19/30 Write block 592 len 16384 data: 13 13 13 13
198382024-02-20T22:49:19.728Z20/30 Write block 302 len 28672 data: 20 21 19 20 15 16 14
198392024-02-20T22:49:19.728Z21/30 Read block 590 len 12288
198402024-02-20T22:49:19.728Z22/30 Write block 140 len 4096 data: 15
198412024-02-20T22:49:19.728Z23/30 Write block 176 len 8192 data: 16 16
198422024-02-20T22:49:19.728Z24/30 Write block 377 len 4096 data: 18
198432024-02-20T22:49:19.728Z25/30 Read block 390 len 40960
198442024-02-20T22:49:19.728Z26/30 Write block 552 len 4096 data: 8
198452024-02-20T22:49:19.734Z27/30 Read block 317 len 24576
198462024-02-20T22:49:19.734Z28/30 Read block 311 len 36864
198472024-02-20T22:49:19.734Z29/30 Read block 587 len 20480
198482024-02-20T22:49:19.734Z30/30 Read block 76 len 24576
198492024-02-20T22:49:19.734Z----------------------------------------------------------------
198502024-02-20T22:49:19.734Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:28
198512024-02-20T22:49:19.734ZStates: Active Active Active
198522024-02-20T22:49:19.734ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198532024-02-20T22:49:19.734Z 3 Acked 1002 Write 8 Done Sent Done false
198542024-02-20T22:49:19.734Z 4 Acked 1003 Flush 0 Done Sent Done false
198552024-02-20T22:49:19.734Z 5 Acked 1004 Write 9 Done Sent Done false
198562024-02-20T22:49:19.734Z 6 Acked 1005 Write 1 Done Sent Done false
198572024-02-20T22:49:19.734Z 7 Acked 1006 Write 9 Done Sent Done false
198582024-02-20T22:49:19.734Z 8 Acked 1007 Read 10 Done Sent Done false
198592024-02-20T22:49:19.734Z 9 Acked 1008 Read 9 Done Sent Done false
198602024-02-20T22:49:19.734Z 10 Acked 1009 Read 6 Done Sent Done false
198612024-02-20T22:49:19.734Z 11 Acked 1010 Flush 0 Done Sent Done false
198622024-02-20T22:49:19.735Z 12 Acked 1011 Write 4 Done Sent Done false
198632024-02-20T22:49:19.735Z 13 Acked 1012 Write 5 Done Sent Done false
198642024-02-20T22:49:19.735Z 14 Acked 1013 Read 4 Done Sent Done false
198652024-02-20T22:49:19.735Z 15 Acked 1014 Write 2 Done Sent Done false
198662024-02-20T22:49:19.735Z 16 Acked 1015 Write 4 Done Sent Done false
198672024-02-20T22:49:19.735Z 17 Acked 1016 Read 3 Done Sent Done false
198682024-02-20T22:49:19.735Z 18 Acked 1017 Read 2 Done Sent Done false
198692024-02-20T22:49:19.735Z 19 Acked 1018 Write 4 Done Sent Done false
198702024-02-20T22:49:19.735Z 20 Acked 1019 Write 7 Done Sent Done false
198712024-02-20T22:49:19.735Z 21 Acked 1020 Read 3 Done Sent Done false
198722024-02-20T22:49:19.735Z 22 Acked 1021 Write 1 Done Sent Done false
198732024-02-20T22:49:19.735Z 23 Acked 1022 Write 2 Done Sent Done false
198742024-02-20T22:49:19.735Z 24 Acked 1023 Write 1 Done Sent Done false
198752024-02-20T22:49:19.735Z 25 Acked 1024 Read 10 Done Sent Done false
198762024-02-20T22:49:19.735Z 26 Acked 1025 Write 1 Done Sent Done false
198772024-02-20T22:49:19.735Z 27 Acked 1026 Read 6 Done Sent Done false
198782024-02-20T22:49:19.735Z 28 Acked 1027 Read 9 Done Sent Done false
198792024-02-20T22:49:19.735Z 29 Acked 1028 Read 5 Done Sent Done false
198802024-02-20T22:49:19.735Z 30 Acked 1029 Read 6 Sent Sent Done false
198812024-02-20T22:49:19.735Z STATES DS:0 DS:1 DS:2 TOTAL
198822024-02-20T22:49:19.735Z New 0 0 0 0
198832024-02-20T22:49:19.735Z Sent 1 28 0 29
198842024-02-20T22:49:19.735Z Done 27 0 28 55
198852024-02-20T22:49:19.735Z Skipped 0 0 0 0
198862024-02-20T22:49:19.736Z Error 0 0 0 0
198872024-02-20T22:49:19.736ZLast Flush: 1010 1001 1010
198882024-02-20T22:49:19.736ZDownstairs last five completed: 1001 1000
198892024-02-20T22:49:19.736ZUpstairs last five completed: 30 29 28 27 26
198902024-02-20T22:49:19.736Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:19.236498232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1874}
198912024-02-20T22:49:19.736ZWrote out file "/tmp/test_repair_verify.data"
198922024-02-20T22:49:19.744Z
198932024-02-20T22:49:19.744ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198942024-02-20T22:49:19.763ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198952024-02-20T22:49:19.763Z 0 000-019 183 183 183 313 313 313 F F F
198962024-02-20T22:49:19.763Z 1 020-039 184 180 184 316 307 316 F F F <---
198972024-02-20T22:49:19.763Z 2 040-059 182 182 182 309 309 309 F F F
198982024-02-20T22:49:19.763Z 3 060-079 175 175 175 300 300 300 F F F
198992024-02-20T22:49:19.763Z 4 080-099 182 182 182 311 311 311 F F F
199002024-02-20T22:49:19.763Z 5 100-119 180 180 180 307 307 307 F F F
199012024-02-20T22:49:19.763Z 6 120-139 183 183 183 313 313 313 F F F
199022024-02-20T22:49:19.763Z 7 140-159 182 182 182 311 311 311 T F T <---
199032024-02-20T22:49:19.763Z 8 160-179 184 184 184 314 314 314 T F T <---
199042024-02-20T22:49:19.767Z 9 180-199 184 184 184 314 314 314 F F F
199052024-02-20T22:49:19.768Z 10 200-219 182 182 182 311 311 311 F F F
199062024-02-20T22:49:19.768Z 11 220-239 180 180 180 307 307 307 F F F
199072024-02-20T22:49:19.768Z 12 240-259 180 180 180 307 307 307 F F F
199082024-02-20T22:49:19.768Z 13 260-279 178 178 178 305 305 305 F F F
199092024-02-20T22:49:19.768Z 14 280-299 178 178 178 305 305 305 F F F
199102024-02-20T22:49:19.768Z 15 300-319 182 182 182 310 310 310 T F T <---
199112024-02-20T22:49:19.768Z 16 320-339 182 182 182 310 310 310 F F F
199122024-02-20T22:49:19.768Z 17 340-359 180 180 180 307 307 307 F F F
199132024-02-20T22:49:19.768Z 18 360-379 180 180 180 307 307 307 T F T <---
199142024-02-20T22:49:19.768Z 19 380-399 180 180 180 307 307 307 T F T <---
199152024-02-20T22:49:19.768Z 20 400-419 182 182 182 309 309 309 F F F
199162024-02-20T22:49:19.768Z 21 420-439 182 182 182 312 312 312 F F F
199172024-02-20T22:49:19.768Z 22 440-459 182 182 182 312 312 312 F F F
199182024-02-20T22:49:19.768Z 23 460-479 184 182 184 315 308 315 F F F <---
199192024-02-20T22:49:19.768Z 24 480-499 184 183 184 316 313 316 F F F <---
199202024-02-20T22:49:19.768Z 25 500-519 180 180 180 307 307 307 T F T <---
199212024-02-20T22:49:19.768Z 26 520-539 184 178 184 316 303 316 T F T <---
199222024-02-20T22:49:19.768Z 27 540-559 180 180 180 307 307 307 T F T <---
199232024-02-20T22:49:19.768Z 28 560-579 176 176 176 301 301 301 F F F
199242024-02-20T22:49:19.768Z 29 580-599 180 180 180 307 307 307 T F T <---
199252024-02-20T22:49:19.768ZMax gen: 184, Max flush: 316
199262024-02-20T22:49:19.769ZError: Difference in extent metadata found!
199272024-02-20T22:49:19.769ZA Difference in extent metadata is expected here
199282024-02-20T22:49:19.769ZOn loop 92
199292024-02-20T22:49:19.769Z
199302024-02-20T22:49:19.769ZVerifying data now
199312024-02-20T22:49:24.371ZNo change after verify
199322024-02-20T22:49:24.372ZLoop: 92 Downstairs dump after verify (and repair):
199332024-02-20T22:49:24.386ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199342024-02-20T22:49:24.386Z 0 000-019 183 183 183 313 313 313 F F F
199352024-02-20T22:49:24.386Z 1 020-039 184 184 184 316 316 316 F F F
199362024-02-20T22:49:24.386Z 2 040-059 182 182 182 309 309 309 F F F
199372024-02-20T22:49:24.386Z 3 060-079 175 175 175 300 300 300 F F F
199382024-02-20T22:49:24.386Z 4 080-099 182 182 182 311 311 311 F F F
199392024-02-20T22:49:24.386Z 5 100-119 180 180 180 307 307 307 F F F
199402024-02-20T22:49:24.386Z 6 120-139 183 183 183 313 313 313 F F F
199412024-02-20T22:49:24.386Z 7 140-159 185 185 185 317 317 317 F F F
199422024-02-20T22:49:24.386Z 8 160-179 185 185 185 317 317 317 F F F
199432024-02-20T22:49:24.386Z 9 180-199 184 184 184 314 314 314 F F F
199442024-02-20T22:49:24.386Z 10 200-219 182 182 182 311 311 311 F F F
199452024-02-20T22:49:24.386Z 11 220-239 180 180 180 307 307 307 F F F
199462024-02-20T22:49:24.386Z 12 240-259 180 180 180 307 307 307 F F F
199472024-02-20T22:49:24.386Z 13 260-279 178 178 178 305 305 305 F F F
199482024-02-20T22:49:24.390Z 14 280-299 178 178 178 305 305 305 F F F
199492024-02-20T22:49:24.390Z 15 300-319 185 185 185 317 317 317 F F F
199502024-02-20T22:49:24.390Z 16 320-339 182 182 182 310 310 310 F F F
199512024-02-20T22:49:24.390Z 17 340-359 180 180 180 307 307 307 F F F
199522024-02-20T22:49:24.390Z 18 360-379 185 185 185 317 317 317 F F F
199532024-02-20T22:49:24.390Z 19 380-399 185 185 185 317 317 317 F F F
199542024-02-20T22:49:24.390Z 20 400-419 182 182 182 309 309 309 F F F
199552024-02-20T22:49:24.390Z 21 420-439 182 182 182 312 312 312 F F F
199562024-02-20T22:49:24.390Z 22 440-459 182 182 182 312 312 312 F F F
199572024-02-20T22:49:24.390Z 23 460-479 184 184 184 315 315 315 F F F
199582024-02-20T22:49:24.390Z 24 480-499 184 184 184 316 316 316 F F F
199592024-02-20T22:49:24.390Z 25 500-519 185 185 185 317 317 317 F F F
199602024-02-20T22:49:24.390Z 26 520-539 185 185 185 317 317 317 F F F
199612024-02-20T22:49:24.390Z 27 540-559 185 185 185 317 317 317 F F F
199622024-02-20T22:49:24.390Z 28 560-579 176 176 176 301 301 301 F F F
199632024-02-20T22:49:24.390Z 29 580-599 185 185 185 317 317 317 F F F
199642024-02-20T22:49:24.390ZMax gen: 185, Max flush: 317
199652024-02-20T22:49:24.390Z
199662024-02-20T22:49:24.390ZBegin loop 93
199672024-02-20T22:49:24.391ZDownstairs to restart: 2
199682024-02-20T22:49:24.409Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.912562718Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882}
199692024-02-20T22:49:24.409Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.912846358Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882}
199702024-02-20T22:49:24.418Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
199712024-02-20T22:49:24.418Z2024-02-20T22:49:23.912889966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882}
199722024-02-20T22:49:24.418Z{"msg":"Crucible 7a9bb6af-f2a5-4648-8ce5-2e7b1936273a has session id: 83d73361-70f2-45f6-bf46-3b9abfcbc297","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.912924985Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199732024-02-20T22:49:24.418Z{"msg":"Upstairs opts: Upstairs UUID: 7a9bb6af-f2a5-4648-8ce5-2e7b1936273a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.912963724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199742024-02-20T22:49:24.418Z{"msg":"Crucible stats registered with UUID: 7a9bb6af-f2a5-4648-8ce5-2e7b1936273a","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913020792Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199752024-02-20T22:49:24.418Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913040581Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882}
199762024-02-20T22:49:24.418Z{"msg":"7a9bb6af-f2a5-4648-8ce5-2e7b1936273a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913167907Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199772024-02-20T22:49:24.418Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913194886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199782024-02-20T22:49:24.418Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913215525Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199792024-02-20T22:49:24.418Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913256363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199802024-02-20T22:49:24.418Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913296572Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199812024-02-20T22:49:24.418Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913322991Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199822024-02-20T22:49:24.418Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.91335592Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199832024-02-20T22:49:24.418Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913393528Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199842024-02-20T22:49:24.418Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913428617Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199852024-02-20T22:49:24.418Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913509244Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199862024-02-20T22:49:24.418Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913585781Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199872024-02-20T22:49:24.418Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913692588Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199882024-02-20T22:49:24.418Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.913721587Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199892024-02-20T22:49:24.418Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914283636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199902024-02-20T22:49:24.418Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914319905Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199912024-02-20T22:49:24.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914342464Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199922024-02-20T22:49:24.419Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914364984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199932024-02-20T22:49:24.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914379203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199942024-02-20T22:49:24.419Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914392463Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199952024-02-20T22:49:24.419Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.914958832Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199962024-02-20T22:49:24.419Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915110737Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199972024-02-20T22:49:24.419Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915199554Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199982024-02-20T22:49:24.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915430506Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
199992024-02-20T22:49:24.419Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915478984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200002024-02-20T22:49:24.419Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915515762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200012024-02-20T22:49:24.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915659097Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200022024-02-20T22:49:24.419Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915718575Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200032024-02-20T22:49:24.419Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915781433Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200042024-02-20T22:49:24.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915807222Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200052024-02-20T22:49:24.419Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915830871Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200062024-02-20T22:49:24.419Z{"msg":"[0]R flush_numbers[0..12]: [313, 316, 309, 300, 311, 307, 313, 317, 317, 314, 311, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.91586394Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200072024-02-20T22:49:24.419Z{"msg":"[0]R generation[0..12]: [183, 184, 182, 175, 182, 180, 183, 185, 185, 184, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915895409Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200082024-02-20T22:49:24.419Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915930848Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200092024-02-20T22:49:24.419Z{"msg":"[1]R flush_numbers[0..12]: [313, 316, 309, 300, 311, 307, 313, 317, 317, 314, 311, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915952027Z","hostname"Wait for a query_work_queue command to finish before sending IO
200102024-02-20T22:49:24.419Z:"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200112024-02-20T22:49:24.419Z{"msg":"[1]R generation[0..12]: [183, 184, 182, 175, 182, 180, 183, 185, 185, 184, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915982776Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200122024-02-20T22:49:24.419Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.915998385Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200132024-02-20T22:49:24.419Z{"msg":"[2]R flush_numbers[0..12]: [313, 316, 309, 300, 311, 307, 313, 317, 317, 314, 311, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916018694Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200142024-02-20T22:49:24.419Z{"msg":"[2]R generation[0..12]: [183, 184, 182, 175, 182, 180, 183, 185, 185, 184, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916055643Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200152024-02-20T22:49:24.420Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916092462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200162024-02-20T22:49:24.420Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916126921Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200172024-02-20T22:49:24.420Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.9161489Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200182024-02-20T22:49:24.420Z{"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916179619Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200192024-02-20T22:49:24.420Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916224727Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200202024-02-20T22:49:24.420Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916257286Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200212024-02-20T22:49:24.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916283685Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200222024-02-20T22:49:24.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916317234Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200232024-02-20T22:49:24.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916355492Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200242024-02-20T22:49:24.420Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916393231Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200252024-02-20T22:49:24.420Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.91643128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200262024-02-20T22:49:24.420Z{"msg":"7a9bb6af-f2a5-4648-8ce5-2e7b1936273a is now active with session: 83d73361-70f2-45f6-bf46-3b9abfcbc297","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916472908Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200272024-02-20T22:49:24.420Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916497607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882,"session_id":"83d73361-70f2-45f6-bf46-3b9abfcbc297"}
200282024-02-20T22:49:24.420Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:23.916533026Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882}
200292024-02-20T22:49:24.420ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200302024-02-20T22:49:24.420ZLoading write count information from file "/tmp/test_repair_verify.data"
200312024-02-20T22:49:24.420ZRun Repair workload
200322024-02-20T22:49:24.430Z01/30 Write block 195 len 36864 data: 15 13 14 13 13 8 7 9 11
200332024-02-20T22:49:24.430Z02/30 Flush
200342024-02-20T22:49:24.430Z03/30 Write block 436 len 8192 data: 8 8
200352024-02-20T22:49:24.430Z04/30 Write block 456 len 4096 data: 21
200362024-02-20T22:49:24.430Z05/30 Read block 414 len 8192
200372024-02-20T22:49:24.430Z06/30 Read block 560 len 20480
200382024-02-20T22:49:24.431Z07/30 Flush
200392024-02-20T22:49:24.431Z08/30 Write block 395 len 32768 data: 11 14 16 10 13 15 12 14
200402024-02-20T22:49:24.431Z09/30 Write block 269 len 36864 data: 11 9 10 10 8 7 11 9 11
200412024-02-20T22:49:24.431Z10/30 Read block 566 len 8192
200422024-02-20T22:49:24.436Z11/30 Read block 519 len 8192
200432024-02-20T22:49:24.436Z12/30 Read block 590 len 8192
200442024-02-20T22:49:24.436Z13/30 Read block 590 len 36864
200452024-02-20T22:49:24.436Z14/30 Read block 462 len 8192
200462024-02-20T22:49:24.436Z15/30 Read block 234 len 32768
200472024-02-20T22:49:24.437Z16/30 Read block 517 len 4096
200482024-02-20T22:49:24.437Z17/30 Read block 224 len 36864
200492024-02-20T22:49:24.437Z18/30 Write block 116 len 40960 data: 15 16 12 15 15 18 20 18 17 13
200502024-02-20T22:49:24.437Z19/30 Flush
200512024-02-20T22:49:24.443Z20/30 Write block 427 len 32768 data: 10 11 11 10 9 6 8 8
200522024-02-20T22:49:24.443Z21/30 Read block 303 len 40960
200532024-02-20T22:49:24.443Z22/30 Read block 348 len 28672
200542024-02-20T22:49:24.443Z23/30 Flush
200552024-02-20T22:49:24.443Z24/30 Write block 401 len 20480 data: 13 15 17 18 16
200562024-02-20T22:49:24.443Z25/30 Read block 333 len 20480
200572024-02-20T22:49:24.447Z26/30 Flush
200582024-02-20T22:49:24.447Z27/30 Write block 224 len 28672 data: 10 8 10 12 15 18 22
200592024-02-20T22:49:24.447Z28/30 Read block 577 len 12288
200602024-02-20T22:49:24.447Z29/30 Read block 95 len 4096
200612024-02-20T22:49:24.462Z30/30 Write block 62 len 16384 data: 12 14 12 14
200622024-02-20T22:49:24.462Z----------------------------------------------------------------
200632024-02-20T22:49:24.462Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:28
200642024-02-20T22:49:24.462ZStates: Active Active Active
200652024-02-20T22:49:24.462ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200662024-02-20T22:49:24.462Z 3 Acked 1002 Write 2 Done Done Sent false
200672024-02-20T22:49:24.462Z 4 Acked 1003 Write 1 Done Done Sent false
200682024-02-20T22:49:24.462Z 5 Acked 1004 Read 2 Done Done Sent false
200692024-02-20T22:49:24.462Z 6 Acked 1005 Read 5 Done Done Sent false
200702024-02-20T22:49:24.462Z 7 Acked 1006 Flush 0 Done Done Sent false
200712024-02-20T22:49:24.462Z 8 Acked 1007 Write 8 Done Done Sent false
200722024-02-20T22:49:24.462Z 9 Acked 1008 Write 9 Done Done Sent false
200732024-02-20T22:49:24.462Z 10 Acked 1009 Read 2 Done Done Sent false
200742024-02-20T22:49:24.462Z 11 Acked 1010 Read 2 Done Done Sent false
200752024-02-20T22:49:24.462Z 12 Acked 1011 Read 2 Done Done Sent false
200762024-02-20T22:49:24.463Z 13 Acked 1012 Read 9 Done Done Sent false
200772024-02-20T22:49:24.463Z 14 Acked 1013 Read 2 Done Done Sent false
200782024-02-20T22:49:24.463Z 15 Acked 1014 Read 8 Done Done Sent false
200792024-02-20T22:49:24.463Z 16 Acked 1015 Read 1 Done Done Sent false
200802024-02-20T22:49:24.463Z 17 Acked 1016 Read 9 Done Done Sent false
200812024-02-20T22:49:24.463Z 18 Acked 1017 Write 10 Done Done Sent false
200822024-02-20T22:49:24.463Z 19 Acked 1018 Flush 0 Done Done Sent false
200832024-02-20T22:49:24.463Z 20 Acked 1019 Write 8 Done Done Sent false
200842024-02-20T22:49:24.463Z 21 Acked 1020 Read 10 Done Done Sent false
200852024-02-20T22:49:24.463Z 22 Acked 1021 Read 7 Done Done Sent false
200862024-02-20T22:49:24.463Z 23 Acked 1022 Flush 0 Done Done Sent false
200872024-02-20T22:49:24.463Z 24 Acked 1023 Write 5 Done Done Sent false
200882024-02-20T22:49:24.463Z 25 Acked 1024 Read 5 Done Done Sent false
200892024-02-20T22:49:24.463Z 26 Acked 1025 Flush 0 Done Done Sent false
200902024-02-20T22:49:24.463Z 27 Acked 1026 Write 7 Done Done Sent false
200912024-02-20T22:49:24.463Z 28 Acked 1027 Read 3 Done Done Sent false
200922024-02-20T22:49:24.463Z 29 Acked 1028 Read 1 Done Done Sent false
200932024-02-20T22:49:24.463Z 30 Acked 1029 Write 4 Sent Sent Sent false
200942024-02-20T22:49:24.463Z STATES DS:0 DS:1 DS:2 TOTAL
200952024-02-20T22:49:24.463Z New 0 0 0 0
200962024-02-20T22:49:24.463Z Sent 1 1 28 30
200972024-02-20T22:49:24.463Z Done 27 27 0 54
200982024-02-20T22:49:24.463Z Skipped 0 0 0 0
200992024-02-20T22:49:24.463Z Error 0 0 0 0
201002024-02-20T22:49:24.464ZLast Flush: 1025 1025 1001
201012024-02-20T22:49:24.464ZDownstairs last five completed: 1001 1000
201022024-02-20T22:49:24.464ZUpstairs last five completed: 30 29 28 27 26
201032024-02-20T22:49:24.464Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:23.951715897Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1882}
201042024-02-20T22:49:24.464ZWrote out file "/tmp/test_repair_verify.data"
201052024-02-20T22:49:24.473Z
201062024-02-20T22:49:24.473ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201072024-02-20T22:49:24.480ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201082024-02-20T22:49:24.480Z 0 000-019 183 183 183 313 313 313 F F F
201092024-02-20T22:49:24.480Z 1 020-039 184 184 184 316 316 316 F F F
201102024-02-20T22:49:24.480Z 2 040-059 182 182 182 309 309 309 F F F
201112024-02-20T22:49:24.480Z 3 060-079 175 175 175 300 300 300 T T F <---
201122024-02-20T22:49:24.480Z 4 080-099 182 182 182 311 311 311 F F F
201132024-02-20T22:49:24.480Z 5 100-119 186 186 180 320 320 307 F F F <---
201142024-02-20T22:49:24.480Z 6 120-139 186 186 183 320 320 313 F F F <---
201152024-02-20T22:49:24.480Z 7 140-159 185 185 185 317 317 317 F F F
201162024-02-20T22:49:24.481Z 8 160-179 185 185 185 317 317 317 F F F
201172024-02-20T22:49:24.481Z 9 180-199 186 186 186 318 318 318 F F F
201182024-02-20T22:49:24.481Z 10 200-219 186 186 186 318 318 318 F F F
201192024-02-20T22:49:24.481Z 11 220-239 180 180 180 307 307 307 T T F <---
201202024-02-20T22:49:24.481Z 12 240-259 180 180 180 307 307 307 F F F
201212024-02-20T22:49:24.481Z 13 260-279 186 186 178 320 320 305 F F F <---
201222024-02-20T22:49:24.481Z 14 280-299 178 178 178 305 305 305 F F F
201232024-02-20T22:49:24.481Z 15 300-319 185 185 185 317 317 317 F F F
201242024-02-20T22:49:24.481Z 16 320-339 182 182 182 310 310 310 F F F
201252024-02-20T22:49:24.481Z 17 340-359 180 180 180 307 307 307 F F F
201262024-02-20T22:49:24.481Z 18 360-379 185 185 185 317 317 317 F F F
201272024-02-20T22:49:24.481Z 19 380-399 186 186 185 320 320 317 F F F <---
201282024-02-20T22:49:24.481Z 20 400-419 186 186 182 322 322 309 F F F <---
201292024-02-20T22:49:24.481Z 21 420-439 186 186 182 321 321 312 F F F <---
201302024-02-20T22:49:24.481Z 22 440-459 186 186 182 319 319 312 F F F <---
201312024-02-20T22:49:24.481Z 23 460-479 184 184 184 315 315 315 F F F
201322024-02-20T22:49:24.481Z 24 480-499 184 184 184 316 316 316 F F F
201332024-02-20T22:49:24.481Z 25 500-519 185 185 185 317 317 317 F F F
201342024-02-20T22:49:24.481Z 26 520-539 185 185 185 317 317 317 F F F
201352024-02-20T22:49:24.481Z 27 540-559 185 185 185 317 317 317 F F F
201362024-02-20T22:49:24.481Z 28 560-579 176 176 176 301 301 301 F F F
201372024-02-20T22:49:24.481Z 29 580-599 185 185 185 317 317 317 F F F
201382024-02-20T22:49:24.481ZMax gen: 186, Max flush: 322
201392024-02-20T22:49:24.481ZError: Difference in extent metadata found!
201402024-02-20T22:49:24.482ZA Difference in extent metadata is expected here
201412024-02-20T22:49:24.482ZOn loop 93
201422024-02-20T22:49:24.482Z
201432024-02-20T22:49:24.482ZVerifying data now
201442024-02-20T22:49:28.958ZNo change after verify
201452024-02-20T22:49:28.958ZLoop: 93 Downstairs dump after verify (and repair):
201462024-02-20T22:49:28.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201472024-02-20T22:49:28.971Z 0 000-019 183 183 183 313 313 313 F F F
201482024-02-20T22:49:28.971Z 1 020-039 184 184 184 316 316 316 F F F
201492024-02-20T22:49:28.971Z 2 040-059 182 182 182 309 309 309 F F F
201502024-02-20T22:49:28.971Z 3 060-079 187 187 187 323 323 323 F F F
201512024-02-20T22:49:28.971Z 4 080-099 182 182 182 311 311 311 F F F
201522024-02-20T22:49:28.971Z 5 100-119 186 186 186 320 320 320 F F F
201532024-02-20T22:49:28.971Z 6 120-139 186 186 186 320 320 320 F F F
201542024-02-20T22:49:28.976Z 7 140-159 185 185 185 317 317 317 F F F
201552024-02-20T22:49:28.976Z 8 160-179 185 185 185 317 317 317 F F F
201562024-02-20T22:49:28.976Z 9 180-199 186 186 186 318 318 318 F F F
201572024-02-20T22:49:28.976Z 10 200-219 186 186 186 318 318 318 F F F
201582024-02-20T22:49:28.976Z 11 220-239 187 187 187 323 323 323 F F F
201592024-02-20T22:49:28.976Z 12 240-259 180 180 180 307 307 307 F F F
201602024-02-20T22:49:28.976Z 13 260-279 186 186 186 320 320 320 F F F
201612024-02-20T22:49:28.976Z 14 280-299 178 178 178 305 305 305 F F F
201622024-02-20T22:49:28.976Z 15 300-319 185 185 185 317 317 317 F F F
201632024-02-20T22:49:28.976Z 16 320-339 182 182 182 310 310 310 F F F
201642024-02-20T22:49:28.976Z 17 340-359 180 180 180 307 307 307 F F F
201652024-02-20T22:49:28.976Z 18 360-379 185 185 185 317 317 317 F F F
201662024-02-20T22:49:28.976Z 19 380-399 186 186 186 320 320 320 F F F
201672024-02-20T22:49:28.976Z 20 400-419 186 186 186 322 322 322 F F F
201682024-02-20T22:49:28.976Z 21 420-439 186 186 186 321 321 321 F F F
201692024-02-20T22:49:28.976Z 22 440-459 186 186 186 319 319 319 F F F
201702024-02-20T22:49:28.976Z 23 460-479 184 184 184 315 315 315 F F F
201712024-02-20T22:49:28.976Z 24 480-499 184 184 184 316 316 316 F F F
201722024-02-20T22:49:28.976Z 25 500-519 185 185 185 317 317 317 F F F
201732024-02-20T22:49:28.977Z 26 520-539 185 185 185 317 317 317 F F F
201742024-02-20T22:49:28.977Z 27 540-559 185 185 185 317 317 317 F F F
201752024-02-20T22:49:28.977Z 28 560-579 176 176 176 301 301 301 F F F
201762024-02-20T22:49:28.977Z 29 580-599 185 185 185 317 317 317 F F F
201772024-02-20T22:49:28.977ZMax gen: 187, Max flush: 323
201782024-02-20T22:49:28.977Z
201792024-02-20T22:49:28.977ZBegin loop 94
201802024-02-20T22:49:28.977ZDownstairs to restart: 1
201812024-02-20T22:49:28.992Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.494929807Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890}
201822024-02-20T22:49:28.992Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495233267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890}
201832024-02-20T22:49:29.001Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
201842024-02-20T22:49:29.001Z,"time":"2024-02-20T22:49:28.495267555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890}
201852024-02-20T22:49:29.001Z{"msg":"Crucible 873852c0-377b-4912-ad4b-7d635ca9d5f7 has session id: 2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495354282Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201862024-02-20T22:49:29.001Z{"msg":"Upstairs opts: Upstairs UUID: 873852c0-377b-4912-ad4b-7d635ca9d5f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.49541921Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201872024-02-20T22:49:29.002Z{"msg":"Crucible stats registered with UUID: 873852c0-377b-4912-ad4b-7d635ca9d5f7","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495452899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201882024-02-20T22:49:29.002Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495471938Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890}
201892024-02-20T22:49:29.002Z{"msg":"873852c0-377b-4912-ad4b-7d635ca9d5f7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495495727Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201902024-02-20T22:49:29.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495536266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201912024-02-20T22:49:29.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495591984Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201922024-02-20T22:49:29.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495627833Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201932024-02-20T22:49:29.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495674291Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201942024-02-20T22:49:29.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495839745Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201952024-02-20T22:49:29.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495892153Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201962024-02-20T22:49:29.002Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495920662Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"io task","client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201972024-02-20T22:49:29.002Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.495959421Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201982024-02-20T22:49:29.002Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.49598681Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"io task","client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
201992024-02-20T22:49:29.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496024318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"io task","client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202002024-02-20T22:49:29.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496044178Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202012024-02-20T22:49:29.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496066607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"io task","client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202022024-02-20T22:49:29.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496559779Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202032024-02-20T22:49:29.005Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496609047Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202042024-02-20T22:49:29.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496662996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202052024-02-20T22:49:29.005Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496697064Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202062024-02-20T22:49:29.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496726973Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202072024-02-20T22:49:29.005Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.496767552Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202082024-02-20T22:49:29.005Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497208156Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202092024-02-20T22:49:29.005Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497251804Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202102024-02-20T22:49:29.005Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497302203Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202112024-02-20T22:49:29.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497789895Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202122024-02-20T22:49:29.005Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497843013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202132024-02-20T22:49:29.005Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497867642Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202142024-02-20T22:49:29.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497899551Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202152024-02-20T22:49:29.006Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.49792134Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202162024-02-20T22:49:29.006Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.49794639Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202172024-02-20T22:49:29.006Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497960139Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202182024-02-20T22:49:29.006Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497973899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202192024-02-20T22:49:29.006Z{"msg":"[0]R flush_numbers[0..12]: [313, 316, 309, 323, 311, 320, 320, 317, 317, 318, 318, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.497995658Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202202024-02-20T22:49:29.006Z{"msg":"[0]R generation[0..12]: [183, 184, 182, 187, 182, 186, 186, 185, 185, 186, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498025487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202212024-02-20T22:49:29.006Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498042266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202222024-02-20T22:49:29.006Z{"msg":"[1]R flush_numbers[0..12]: [313, 316, 309, 323, 311, 320, 320, 317, 317, 318, 318, 323]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
202232024-02-20T22:49:29.006Ztime":"2024-02-20T22:49:28.498056526Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202242024-02-20T22:49:29.006Z{"msg":"[1]R generation[0..12]: [183, 184, 182, 187, 182, 186, 186, 185, 185, 186, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498091254Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202252024-02-20T22:49:29.006Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498121713Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202262024-02-20T22:49:29.006Z{"msg":"[2]R flush_numbers[0..12]: [313, 316, 309, 323, 311, 320, 320, 317, 317, 318, 318, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498137103Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202272024-02-20T22:49:29.006Z{"msg":"[2]R generation[0..12]: [183, 184, 182, 187, 182, 186, 186, 185, 185, 186, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498151462Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202282024-02-20T22:49:29.006Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498165522Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202292024-02-20T22:49:29.007Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498185841Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202302024-02-20T22:49:29.007Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498199721Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202312024-02-20T22:49:29.007Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.49821374Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202322024-02-20T22:49:29.007Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498233509Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202332024-02-20T22:49:29.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498263278Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202342024-02-20T22:49:29.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498277198Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202352024-02-20T22:49:29.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498307607Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202362024-02-20T22:49:29.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498325676Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202372024-02-20T22:49:29.008Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498355705Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202382024-02-20T22:49:29.008Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498370324Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202392024-02-20T22:49:29.008Z{"msg":"873852c0-377b-4912-ad4b-7d635ca9d5f7 is now active with session: 2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498391934Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202402024-02-20T22:49:29.008Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498415793Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890,"session_id":"2fd7eb54-3fbf-4caf-85e0-0cf3ce5fb1d8"}
202412024-02-20T22:49:29.008Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:28.498429912Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890}
202422024-02-20T22:49:29.008ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202432024-02-20T22:49:29.008ZLoading write count information from file "/tmp/test_repair_verify.data"
202442024-02-20T22:49:29.008ZRun Repair workload
202452024-02-20T22:49:29.008Z01/30 Write block 95 len 16384 data: 12 9 13 10
202462024-02-20T22:49:29.025Z02/30 Write block 326 len 32768 data: 19 19 18 16 14 13 18 19
202472024-02-20T22:49:29.025Z03/30 Write block 236 len 36864 data: 20 19 15 14 11 12 10 10 8
202482024-02-20T22:49:29.025Z04/30 Write block 467 len 8192 data: 20 23
202492024-02-20T22:49:29.025Z05/30 Write block 112 len 24576 data: 15 16 14 13 16 17
202502024-02-20T22:49:29.025Z06/30 Read block 339 len 4096
202512024-02-20T22:49:29.025Z07/30 Flush
202522024-02-20T22:49:29.026Z08/30 Write block 298 len 20480 data: 14 15 13 18 21
202532024-02-20T22:49:29.026Z09/30 Write block 333 len 12288 data: 20 17 14
202542024-02-20T22:49:29.026Z10/30 Read block 101 len 32768
202552024-02-20T22:49:29.026Z11/30 Write block 191 len 36864 data: 13 13 12 11 16 14 15 14 14
202562024-02-20T22:49:29.026Z12/30 Write block 243 len 24576 data: 11 9 8 9 12 11
202572024-02-20T22:49:29.026Z13/30 Write block 92 len 8192 data: 10 11
202582024-02-20T22:49:29.026Z14/30 Write block 594 len 12288 data: 14 14 17
202592024-02-20T22:49:29.026Z15/30 Read block 222 len 32768
202602024-02-20T22:49:29.026Z16/30 Write block 271 len 24576 data: 11 11 9 8 12 10
202612024-02-20T22:49:29.026Z17/30 Write block 530 len 36864 data: 19 19 15 17 18 21 19 17 22
202622024-02-20T22:49:29.026Z18/30 Write block 97 len 40960 data: 14 11 15 13 13 12 11 12 11 11
202632024-02-20T22:49:29.026Z19/30 Write block 226 len 4096 data: 11
202642024-02-20T22:49:29.026Z20/30 Read block 303 len 36864
202652024-02-20T22:49:29.031Z21/30 Write block 23 len 24576 data: 13 12 17 17 22 22
202662024-02-20T22:49:29.031Z22/30 Flush
202672024-02-20T22:49:29.031Z23/30 Write block 101 len 20480 data: 14 13 12 13 12
202682024-02-20T22:49:29.032Z24/30 Read block 105 len 12288
202692024-02-20T22:49:29.032Z25/30 Read block 394 len 16384
202702024-02-20T22:49:29.032Z26/30 Read block 46 len 16384
202712024-02-20T22:49:29.032Z27/30 Read block 397 len 16384
202722024-02-20T22:49:29.032Z28/30 Write block 289 len 40960 data: 10 10 11 8 8 7 6 9 11 15
202732024-02-20T22:49:29.032Z29/30 Read block 0 len 20480
202742024-02-20T22:49:29.032Z30/30 Write block 66 len 4096 data: 13
202752024-02-20T22:49:29.032Z----------------------------------------------------------------
202762024-02-20T22:49:29.032Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202772024-02-20T22:49:29.032ZStates: Active Active Active
202782024-02-20T22:49:29.032ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202792024-02-20T22:49:29.032Z 1 Acked 1000 Write 4 Done Sent Done false
202802024-02-20T22:49:29.032Z 2 Acked 1001 Write 8 Done Sent Done false
202812024-02-20T22:49:29.032Z 3 Acked 1002 Write 9 Done Sent Done false
202822024-02-20T22:49:29.032Z 4 Acked 1003 Write 2 Done Sent Done false
202832024-02-20T22:49:29.032Z 5 Acked 1004 Write 6 Done Sent Done false
202842024-02-20T22:49:29.032Z 6 Acked 1005 Read 1 Done Sent Done false
202852024-02-20T22:49:29.032Z 7 Acked 1006 Flush 0 Done Sent Done false
202862024-02-20T22:49:29.032Z 8 Acked 1007 Write 5 Done Sent Done false
202872024-02-20T22:49:29.033Z 9 Acked 1008 Write 3 Done Sent Done false
202882024-02-20T22:49:29.033Z 10 Acked 1009 Read 8 Done Sent Done false
202892024-02-20T22:49:29.033Z 11 Acked 1010 Write 9 Done Sent Done false
202902024-02-20T22:49:29.033Z 12 Acked 1011 Write 6 Done Sent Done false
202912024-02-20T22:49:29.033Z 13 Acked 1012 Write 2 Done Sent Done false
202922024-02-20T22:49:29.033Z 14 Acked 1013 Write 3 Done Sent Done false
202932024-02-20T22:49:29.033Z 15 Acked 1014 Read 8 Done Sent Done false
202942024-02-20T22:49:29.033Z 16 Acked 1015 Write 6 Done Sent Done false
202952024-02-20T22:49:29.033Z 17 Acked 1016 Write 9 Done Sent Done false
202962024-02-20T22:49:29.033Z 18 Acked 1017 Write 10 Done Sent Done false
202972024-02-20T22:49:29.033Z 19 Acked 1018 Write 1 Done Sent Done false
202982024-02-20T22:49:29.033Z 20 Acked 1019 Read 9 Done Sent Done false
202992024-02-20T22:49:29.033Z 21 Acked 1020 Write 6 Done Sent Done false
203002024-02-20T22:49:29.033Z 22 Acked 1021 Flush 0 Done Sent Done false
203012024-02-20T22:49:29.033Z 23 Acked 1022 Write 5 Done Sent Done false
203022024-02-20T22:49:29.033Z 24 Acked 1023 Read 3 Done Sent Done false
203032024-02-20T22:49:29.033Z 25 Acked 1024 Read 4 Done Sent Done false
203042024-02-20T22:49:29.033Z 26 Acked 1025 Read 4 Done Sent Done false
203052024-02-20T22:49:29.033Z 27 Acked 1026 Read 4 Done Sent Done false
203062024-02-20T22:49:29.034Z 28 Acked 1027 Write 10 Done Sent Done false
203072024-02-20T22:49:29.034Z 29 Acked 1028 Read 5 Done Sent Done false
203082024-02-20T22:49:29.034Z 30 Acked 1029 Write 1 Sent Sent Sent false
203092024-02-20T22:49:29.034Z STATES DS:0 DS:1 DS:2 TOTAL
203102024-02-20T22:49:29.034Z New 0 0 0 0
203112024-02-20T22:49:29.034Z Sent 1 30 1 32
203122024-02-20T22:49:29.034Z Done 29 0 29 58
203132024-02-20T22:49:29.034Z Skipped 0 0 0 0
203142024-02-20T22:49:29.034Z Error 0 0 0 0
203152024-02-20T22:49:29.034ZLast Flush: 1021 0 1021
203162024-02-20T22:49:29.034ZDownstairs last five completed:
203172024-02-20T22:49:29.034ZUpstairs last five completed: 30 29 28 27 26
203182024-02-20T22:49:29.034Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:28.529627886Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1890}
203192024-02-20T22:49:29.034ZWrote out file "/tmp/test_repair_verify.data"
203202024-02-20T22:49:29.034Z
203212024-02-20T22:49:29.039ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203222024-02-20T22:49:29.049ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203232024-02-20T22:49:29.049Z 0 000-019 183 183 183 313 313 313 F F F
203242024-02-20T22:49:29.049Z 1 020-039 188 184 188 325 316 325 F F F <---
203252024-02-20T22:49:29.049Z 2 040-059 182 182 182 309 309 309 F F F
203262024-02-20T22:49:29.049Z 3 060-079 187 187 187 323 323 323 T F T <---
203272024-02-20T22:49:29.049Z 4 080-099 188 182 188 325 311 325 F F F <---
203282024-02-20T22:49:29.049Z 5 100-119 188 186 188 325 320 325 T F T <---
203292024-02-20T22:49:29.049Z 6 120-139 186 186 186 320 320 320 F F F
203302024-02-20T22:49:29.049Z 7 140-159 185 185 185 317 317 317 F F F
203312024-02-20T22:49:29.049Z 8 160-179 185 185 185 317 317 317 F F F
203322024-02-20T22:49:29.049Z 9 180-199 188 186 188 325 318 325 F F F <---
203332024-02-20T22:49:29.049Z 10 200-219 186 186 186 318 318 318 F F F
203342024-02-20T22:49:29.056Z 11 220-239 188 187 188 325 323 325 F F F <---
203352024-02-20T22:49:29.056Z 12 240-259 188 180 188 325 307 325 F F F <---
203362024-02-20T22:49:29.056Z 13 260-279 188 186 188 325 320 325 F F F <---
203372024-02-20T22:49:29.056Z 14 280-299 188 178 188 325 305 325 T F T <---
203382024-02-20T22:49:29.056Z 15 300-319 188 185 188 325 317 325 F F F <---
203392024-02-20T22:49:29.056Z 16 320-339 188 182 188 325 310 325 F F F <---
203402024-02-20T22:49:29.056Z 17 340-359 180 180 180 307 307 307 F F F
203412024-02-20T22:49:29.056Z 18 360-379 185 185 185 317 317 317 F F F
203422024-02-20T22:49:29.056Z 19 380-399 186 186 186 320 320 320 F F F
203432024-02-20T22:49:29.056Z 20 400-419 186 186 186 322 322 322 F F F
203442024-02-20T22:49:29.056Z 21 420-439 186 186 186 321 321 321 F F F
203452024-02-20T22:49:29.056Z 22 440-459 186 186 186 319 319 319 F F F
203462024-02-20T22:49:29.056Z 23 460-479 188 184 188 324 315 324 F F F <---
203472024-02-20T22:49:29.056Z 24 480-499 184 184 184 316 316 316 F F F
203482024-02-20T22:49:29.056Z 25 500-519 185 185 185 317 317 317 F F F
203492024-02-20T22:49:29.056Z 26 520-539 188 185 188 325 317 325 F F F <---
203502024-02-20T22:49:29.056Z 27 540-559 185 185 185 317 317 317 F F F
203512024-02-20T22:49:29.057Z 28 560-579 176 176 176 301 301 301 F F F
203522024-02-20T22:49:29.057Z 29 580-599 188 185 188 325 317 325 F F F <---
203532024-02-20T22:49:29.057ZMax gen: 188, Max flush: 325
203542024-02-20T22:49:29.057ZError: Difference in extent metadata found!
203552024-02-20T22:49:29.057ZA Difference in extent metadata is expected here
203562024-02-20T22:49:29.057ZOn loop 94
203572024-02-20T22:49:29.057Z
203582024-02-20T22:49:29.057ZVerifying data now
203592024-02-20T22:49:33.748ZNo change after verify
203602024-02-20T22:49:33.748ZLoop: 94 Downstairs dump after verify (and repair):
203612024-02-20T22:49:33.760ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203622024-02-20T22:49:33.760Z 0 000-019 183 183 183 313 313 313 F F F
203632024-02-20T22:49:33.765Z 1 020-039 188 188 188 325 325 325 F F F
203642024-02-20T22:49:33.765Z 2 040-059 182 182 182 309 309 309 F F F
203652024-02-20T22:49:33.766Z 3 060-079 189 189 189 326 326 326 F F F
203662024-02-20T22:49:33.766Z 4 080-099 188 188 188 325 325 325 F F F
203672024-02-20T22:49:33.766Z 5 100-119 189 189 189 326 326 326 F F F
203682024-02-20T22:49:33.766Z 6 120-139 186 186 186 320 320 320 F F F
203692024-02-20T22:49:33.766Z 7 140-159 185 185 185 317 317 317 F F F
203702024-02-20T22:49:33.766Z 8 160-179 185 185 185 317 317 317 F F F
203712024-02-20T22:49:33.766Z 9 180-199 188 188 188 325 325 325 F F F
203722024-02-20T22:49:33.766Z 10 200-219 186 186 186 318 318 318 F F F
203732024-02-20T22:49:33.766Z 11 220-239 188 188 188 325 325 325 F F F
203742024-02-20T22:49:33.766Z 12 240-259 188 188 188 325 325 325 F F F
203752024-02-20T22:49:33.766Z 13 260-279 188 188 188 325 325 325 F F F
203762024-02-20T22:49:33.766Z 14 280-299 189 189 189 326 326 326 F F F
203772024-02-20T22:49:33.766Z 15 300-319 188 188 188 325 325 325 F F F
203782024-02-20T22:49:33.766Z 16 320-339 188 188 188 325 325 325 F F F
203792024-02-20T22:49:33.767Z 17 340-359 180 180 180 307 307 307 F F F
203802024-02-20T22:49:33.767Z 18 360-379 185 185 185 317 317 317 F F F
203812024-02-20T22:49:33.767Z 19 380-399 186 186 186 320 320 320 F F F
203822024-02-20T22:49:33.767Z 20 400-419 186 186 186 322 322 322 F F F
203832024-02-20T22:49:33.767Z 21 420-439 186 186 186 321 321 321 F F F
203842024-02-20T22:49:33.767Z 22 440-459 186 186 186 319 319 319 F F F
203852024-02-20T22:49:33.767Z 23 460-479 188 188 188 324 324 324 F F F
203862024-02-20T22:49:33.767Z 24 480-499 184 184 184 316 316 316 F F F
203872024-02-20T22:49:33.767Z 25 500-519 185 185 185 317 317 317 F F F
203882024-02-20T22:49:33.767Z 26 520-539 188 188 188 325 325 325 F F F
203892024-02-20T22:49:33.767Z 27 540-559 185 185 185 317 317 317 F F F
203902024-02-20T22:49:33.767Z 28 560-579 176 176 176 301 301 301 F F F
203912024-02-20T22:49:33.767Z 29 580-599 188 188 188 325 325 325 F F F
203922024-02-20T22:49:33.767ZMax gen: 189, Max flush: 326
203932024-02-20T22:49:33.767Z
203942024-02-20T22:49:33.767ZBegin loop 95
203952024-02-20T22:49:33.767ZDownstairs to restart: 1
203962024-02-20T22:49:33.781Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.283854135Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898}
203972024-02-20T22:49:33.781Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284190493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898}
203982024-02-20T22:49:33.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284235982Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898}
203992024-02-20T22:49:33.789Z{"msg":"Crucible 49fda745-c509-48f4-801e-2b2dfcc8b8c3 has session id: 43ed6fa3-c4bb-4ecc-837a-80fc8816375e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284264881Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"Crucible runtime is spawned
204002024-02-20T22:49:33.789Zsession_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204012024-02-20T22:49:33.789Z{"msg":"Upstairs opts: Upstairs UUID: 49fda745-c509-48f4-801e-2b2dfcc8b8c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284322899Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204022024-02-20T22:49:33.789Z{"msg":"Crucible stats registered with UUID: 49fda745-c509-48f4-801e-2b2dfcc8b8c3","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284356667Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204032024-02-20T22:49:33.789Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284382426Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898}
204042024-02-20T22:49:33.789Z{"msg":"49fda745-c509-48f4-801e-2b2dfcc8b8c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284442724Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204052024-02-20T22:49:33.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284472553Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204062024-02-20T22:49:33.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284491213Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204072024-02-20T22:49:33.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284532971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204082024-02-20T22:49:33.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.28457023Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204092024-02-20T22:49:33.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284600019Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204102024-02-20T22:49:33.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284694515Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204112024-02-20T22:49:33.789Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284731684Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"io task","client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204122024-02-20T22:49:33.789Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284776232Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"io task","client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204132024-02-20T22:49:33.789Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284802711Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"io task","client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204142024-02-20T22:49:33.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.28484044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"io task","client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204152024-02-20T22:49:33.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.284996035Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"io task","client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204162024-02-20T22:49:33.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285037593Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"io task","client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204172024-02-20T22:49:33.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285471858Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204182024-02-20T22:49:33.789Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285499226Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204192024-02-20T22:49:33.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285514796Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204202024-02-20T22:49:33.790Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285543665Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204212024-02-20T22:49:33.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285557854Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204222024-02-20T22:49:33.790Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.285579334Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204232024-02-20T22:49:33.790Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286119624Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204242024-02-20T22:49:33.790Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286159473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204252024-02-20T22:49:33.790Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286175962Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204262024-02-20T22:49:33.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286687064Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204272024-02-20T22:49:33.790Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286723763Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204282024-02-20T22:49:33.790Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286740422Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204292024-02-20T22:49:33.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.28680947Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204302024-02-20T22:49:33.790Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286832569Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204312024-02-20T22:49:33.790Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286847318Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204322024-02-20T22:49:33.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286867468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204332024-02-20T22:49:33.790Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286881847Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204342024-02-20T22:49:33.790Z{"msg":"[0]R flush_numbers[0..12]: [313, 325, 309, 326, 325, 326, 320, 317, 317, 325, 318, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286904176Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204352024-02-20T22:49:33.790Z{"msg":"[0]R generation[0..12]: [183, 188, 182, 189, 188, 189, 186, 185, 185, 188, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286919076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204362024-02-20T22:49:33.790Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286940545Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204372024-02-20T22:49:33.791Z{"msg":"[1]R flush_numbers[0..12]: [313, 325, 309, 326, 325, 326, 320, 317, 317, 325, 318, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286955754Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204382024-02-20T22:49:33.791Z{"msg":"[1]R generation[0..12]: [183, 188, 182, 189, 188, 189, 186, 185, 185, 188, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286977253Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204392024-02-20T22:49:33.791Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.286992363Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204402024-02-20T22:49:33.791Z{"msg":"[2]R flush_numbers[0..12]: [313, 325, 309, 326, 325, 326, 320, 317, 317, 325, 318, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287014012Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204412024-02-20T22:49:33.791Z{"msg":"[2]R generation[0..12]: [183, 188, 182, 189, 188, 189, 186, 185, 185, 188, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287028582Z","hostname":"Wait for a query_work_queue command to finish before sending IO
204422024-02-20T22:49:33.791Zip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204432024-02-20T22:49:33.791Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287055501Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204442024-02-20T22:49:33.791Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.28707004Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204452024-02-20T22:49:33.791Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287091269Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204462024-02-20T22:49:33.791Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287105369Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204472024-02-20T22:49:33.791Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287118499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204482024-02-20T22:49:33.791Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287131998Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204492024-02-20T22:49:33.791Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287145267Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204502024-02-20T22:49:33.791Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287166977Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204512024-02-20T22:49:33.791Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287181216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204522024-02-20T22:49:33.791Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287207135Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204532024-02-20T22:49:33.792Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287220855Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204542024-02-20T22:49:33.792Z{"msg":"49fda745-c509-48f4-801e-2b2dfcc8b8c3 is now active with session: 43ed6fa3-c4bb-4ecc-837a-80fc8816375e","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287240194Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204552024-02-20T22:49:33.792Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287260914Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898,"session_id":"43ed6fa3-c4bb-4ecc-837a-80fc8816375e"}
204562024-02-20T22:49:33.792Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:33.287274653Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898}
204572024-02-20T22:49:33.792ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204582024-02-20T22:49:33.796ZLoading write count information from file "/tmp/test_repair_verify.data"
204592024-02-20T22:49:33.796ZRun Repair workload
204602024-02-20T22:49:33.796Z01/30 Flush
204612024-02-20T22:49:33.796Z02/30 Write block 523 len 8192 data: 12 9
204622024-02-20T22:49:33.797Z03/30 Write block 179 len 4096 data: 19
204632024-02-20T22:49:33.797Z04/30 Write block 70 len 24576 data: 16 16 16 12 11 9
204642024-02-20T22:49:33.797Z05/30 Write block 30 len 4096 data: 26
204652024-02-20T22:49:33.797Z06/30 Write block 568 len 20480 data: 9 7 8 6 5
204662024-02-20T22:49:33.797Z07/30 Write block 426 len 8192 data: 12 11
204672024-02-20T22:49:33.797Z08/30 Write block 118 len 12288 data: 13 16 16
204682024-02-20T22:49:33.797Z09/30 Write block 26 len 16384 data: 18 23 23 26
204692024-02-20T22:49:33.797Z10/30 Flush
204702024-02-20T22:49:33.797Z11/30 Flush
204712024-02-20T22:49:33.797Z12/30 Write block 378 len 24576 data: 15 14 15 12 15 14
204722024-02-20T22:49:33.797Z13/30 Write block 487 len 12288 data: 17 18 15
204732024-02-20T22:49:33.801Z14/30 Write block 13 len 8192 data: 12 11
204742024-02-20T22:49:33.801Z15/30 Read block 307 len 36864
204752024-02-20T22:49:33.801Z16/30 Write block 283 len 32768 data: 12 12 15 14 15 14 11 11
204762024-02-20T22:49:33.801Z17/30 Write block 513 len 24576 data: 15 15 16 14 17 16
204772024-02-20T22:49:33.801Z18/30 Read block 243 len 36864
204782024-02-20T22:49:33.801Z19/30 Flush
204792024-02-20T22:49:33.805Z20/30 Write block 364 len 24576 data: 23 23 22 15 17 20
204802024-02-20T22:49:33.805Z21/30 Read block 31 len 8192
204812024-02-20T22:49:33.809Z22/30 Write block 192 len 32768 data: 14 13 12 17 15 16 15 15
204822024-02-20T22:49:33.809Z23/30 Write block 114 len 28672 data: 15 14 17 18 14 17 17
204832024-02-20T22:49:33.809Z24/30 Write block 48 len 12288 data: 14 15 13
204842024-02-20T22:49:33.809Z25/30 Write block 316 len 36864 data: 15 17 15 15 17 17 20 15 13
204852024-02-20T22:49:33.815Z26/30 Write block 36 len 16384 data: 21 20 18 19
204862024-02-20T22:49:33.815Z27/30 Write block 322 len 24576 data: 21 16 14 18 20 20
204872024-02-20T22:49:33.815Z28/30 Write block 296 len 16384 data: 10 12 16 16
204882024-02-20T22:49:33.815Z29/30 Read block 68 len 40960
204892024-02-20T22:49:33.815Z30/30 Write block 215 len 24576 data: 12 11 8 8 8 9
204902024-02-20T22:49:33.815Z----------------------------------------------------------------
204912024-02-20T22:49:33.815Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:29
204922024-02-20T22:49:33.815ZStates: Active Active Active
204932024-02-20T22:49:33.815ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204942024-02-20T22:49:33.815Z 2 Acked 1001 Write 2 Done Done Done false
204952024-02-20T22:49:33.815Z 3 Acked 1002 Write 1 Done Sent Done false
204962024-02-20T22:49:33.815Z 4 Acked 1003 Write 6 Done Sent Done false
204972024-02-20T22:49:33.815Z 5 Acked 1004 Write 1 Done Sent Done false
204982024-02-20T22:49:33.815Z 6 Acked 1005 Write 5 Done Sent Done false
204992024-02-20T22:49:33.815Z 7 Acked 1006 Write 2 Done Sent Done false
205002024-02-20T22:49:33.815Z 8 Acked 1007 Write 3 Done Sent Done false
205012024-02-20T22:49:33.815Z 9 Acked 1008 Write 4 Done Sent Done false
205022024-02-20T22:49:33.815Z 10 Acked 1009 Flush 0 Done Sent Done false
205032024-02-20T22:49:33.815Z 11 Acked 1010 Flush 0 Done Sent Done false
205042024-02-20T22:49:33.815Z 12 Acked 1011 Write 6 Done Sent Done false
205052024-02-20T22:49:33.815Z 13 Acked 1012 Write 3 Done Sent Done false
205062024-02-20T22:49:33.816Z 14 Acked 1013 Write 2 Done Sent Done false
205072024-02-20T22:49:33.816Z 15 Acked 1014 Read 9 Done Sent Done false
205082024-02-20T22:49:33.816Z 16 Acked 1015 Write 8 Done Sent Done false
205092024-02-20T22:49:33.816Z 17 Acked 1016 Write 6 Done Sent Done false
205102024-02-20T22:49:33.816Z 18 Acked 1017 Read 9 Done Sent Done false
205112024-02-20T22:49:33.816Z 19 Acked 1018 Flush 0 Done Sent Done false
205122024-02-20T22:49:33.816Z 20 Acked 1019 Write 6 Done Sent Done false
205132024-02-20T22:49:33.816Z 21 Acked 1020 Read 2 Done Sent Done false
205142024-02-20T22:49:33.816Z 22 Acked 1021 Write 8 Done Sent Done false
205152024-02-20T22:49:33.816Z 23 Acked 1022 Write 7 Done Sent Done false
205162024-02-20T22:49:33.816Z 24 Acked 1023 Write 3 Done Sent Done false
205172024-02-20T22:49:33.816Z 25 Acked 1024 Write 9 Done Sent Done false
205182024-02-20T22:49:33.816Z 26 Acked 1025 Write 4 Done Sent Done false
205192024-02-20T22:49:33.816Z 27 Acked 1026 Write 6 Done Sent Done false
205202024-02-20T22:49:33.816Z 28 Acked 1027 Write 4 Done Sent Done false
205212024-02-20T22:49:33.816Z 29 Acked 1028 Read 10 Done Sent Done false
205222024-02-20T22:49:33.816Z 30 Acked 1029 Write 6 Sent Sent Sent false
205232024-02-20T22:49:33.816Z STATES DS:0 DS:1 DS:2 TOTAL
205242024-02-20T22:49:33.816Z New 0 0 0 0
205252024-02-20T22:49:33.816Z Sent 1 28 1 30
205262024-02-20T22:49:33.816Z Done 28 1 28 57
205272024-02-20T22:49:33.816Z Skipped 0 0 0 0
205282024-02-20T22:49:33.816Z Error 0 0 0 0
205292024-02-20T22:49:33.816ZLast Flush: 1018 1000 1018
205302024-02-20T22:49:33.816ZDownstairs last five completed: 1000
205312024-02-20T22:49:33.817ZUpstairs last five completed: 30 29 28 27 26
205322024-02-20T22:49:33.817Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:33.316205798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1898}
205332024-02-20T22:49:33.821ZWrote out file "/tmp/test_repair_verify.data"
205342024-02-20T22:49:33.822Z
205352024-02-20T22:49:33.825ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205362024-02-20T22:49:33.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205372024-02-20T22:49:33.840Z 0 000-019 190 183 190 330 313 330 F F F <---
205382024-02-20T22:49:33.840Z 1 020-039 190 188 190 328 325 328 T F T <---
205392024-02-20T22:49:33.840Z 2 040-059 182 182 182 309 309 309 T F T <---
205402024-02-20T22:49:33.840Z 3 060-079 190 189 190 328 326 328 F F F <---
205412024-02-20T22:49:33.840Z 4 080-099 188 188 188 325 325 325 F F F
205422024-02-20T22:49:33.840Z 5 100-119 190 189 190 328 326 328 T F T <---
205432024-02-20T22:49:33.840Z 6 120-139 190 186 190 328 320 328 T F T <---
205442024-02-20T22:49:33.840Z 7 140-159 185 185 185 317 317 317 F F F
205452024-02-20T22:49:33.840Z 8 160-179 190 185 190 328 317 328 F F F <---
205462024-02-20T22:49:33.840Z 9 180-199 188 188 188 325 325 325 T F T <---
205472024-02-20T22:49:33.840Z 10 200-219 186 186 186 318 318 318 T F T <---
205482024-02-20T22:49:33.840Z 11 220-239 188 188 188 325 325 325 T F T <---
205492024-02-20T22:49:33.840Z 12 240-259 188 188 188 325 325 325 F F F
205502024-02-20T22:49:33.844Z 13 260-279 188 188 188 325 325 325 F F F
205512024-02-20T22:49:33.844Z 14 280-299 190 189 190 330 326 330 T F T <---
205522024-02-20T22:49:33.844Z 15 300-319 188 188 188 325 325 325 T F T <---
205532024-02-20T22:49:33.844Z 16 320-339 188 188 188 325 325 325 T F T <---
205542024-02-20T22:49:33.844Z 17 340-359 180 180 180 307 307 307 F F F
205552024-02-20T22:49:33.844Z 18 360-379 190 185 190 330 317 330 T F T <---
205562024-02-20T22:49:33.844Z 19 380-399 190 186 190 330 320 330 F F F <---
205572024-02-20T22:49:33.845Z 20 400-419 186 186 186 322 322 322 F F F
205582024-02-20T22:49:33.845Z 21 420-439 190 186 190 328 321 328 F F F <---
205592024-02-20T22:49:33.845Z 22 440-459 186 186 186 319 319 319 F F F
205602024-02-20T22:49:33.845Z 23 460-479 188 188 188 324 324 324 F F F
205612024-02-20T22:49:33.845Z 24 480-499 190 184 190 330 316 330 F F F <---
205622024-02-20T22:49:33.845Z 25 500-519 190 185 190 330 317 330 F F F <---
205632024-02-20T22:49:33.845Z 26 520-539 190 188 190 328 325 328 F T F <---
205642024-02-20T22:49:33.845Z 27 540-559 185 185 185 317 317 317 F F F
205652024-02-20T22:49:33.845Z 28 560-579 190 176 190 328 301 328 F F F <---
205662024-02-20T22:49:33.845Z 29 580-599 188 188 188 325 325 325 F F F
205672024-02-20T22:49:33.845ZMax gen: 190, Max flush: 330
205682024-02-20T22:49:33.845ZError: Difference in extent metadata found!
205692024-02-20T22:49:33.845ZA Difference in extent metadata is expected here
205702024-02-20T22:49:33.845ZOn loop 95
205712024-02-20T22:49:33.845Z
205722024-02-20T22:49:33.849ZVerifying data now
205732024-02-20T22:49:38.784ZNo change after verify
205742024-02-20T22:49:38.784ZLoop: 95 Downstairs dump after verify (and repair):
205752024-02-20T22:49:38.799ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205762024-02-20T22:49:38.799Z 0 000-019 190 190 190 330 330 330 F F F
205772024-02-20T22:49:38.799Z 1 020-039 191 191 191 331 331 331 F F F
205782024-02-20T22:49:38.799Z 2 040-059 191 191 191 331 331 331 F F F
205792024-02-20T22:49:38.799Z 3 060-079 190 190 190 328 328 328 F F F
205802024-02-20T22:49:38.799Z 4 080-099 188 188 188 325 325 325 F F F
205812024-02-20T22:49:38.799Z 5 100-119 191 191 191 331 331 331 F F F
205822024-02-20T22:49:38.799Z 6 120-139 191 191 191 331 331 331 F F F
205832024-02-20T22:49:38.800Z 7 140-159 185 185 185 317 317 317 F F F
205842024-02-20T22:49:38.800Z 8 160-179 190 190 190 328 328 328 F F F
205852024-02-20T22:49:38.800Z 9 180-199 191 191 191 331 331 331 F F F
205862024-02-20T22:49:38.800Z 10 200-219 191 191 191 331 331 331 F F F
205872024-02-20T22:49:38.800Z 11 220-239 191 191 191 331 331 331 F F F
205882024-02-20T22:49:38.808Z 12 240-259 188 188 188 325 325 325 F F F
205892024-02-20T22:49:38.808Z 13 260-279 188 188 188 325 325 325 F F F
205902024-02-20T22:49:38.808Z 14 280-299 191 191 191 331 331 331 F F F
205912024-02-20T22:49:38.808Z 15 300-319 191 191 191 331 331 331 F F F
205922024-02-20T22:49:38.808Z 16 320-339 191 191 191 331 331 331 F F F
205932024-02-20T22:49:38.808Z 17 340-359 180 180 180 307 307 307 F F F
205942024-02-20T22:49:38.808Z 18 360-379 191 191 191 331 331 331 F F F
205952024-02-20T22:49:38.809Z 19 380-399 190 190 190 330 330 330 F F F
205962024-02-20T22:49:38.809Z 20 400-419 186 186 186 322 322 322 F F F
205972024-02-20T22:49:38.809Z 21 420-439 190 190 190 328 328 328 F F F
205982024-02-20T22:49:38.809Z 22 440-459 186 186 186 319 319 319 F F F
205992024-02-20T22:49:38.809Z 23 460-479 188 188 188 324 324 324 F F F
206002024-02-20T22:49:38.809Z 24 480-499 190 190 190 330 330 330 F F F
206012024-02-20T22:49:38.809Z 25 500-519 190 190 190 330 330 330 F F F
206022024-02-20T22:49:38.809Z 26 520-539 190 190 190 328 328 328 F F F
206032024-02-20T22:49:38.809Z 27 540-559 185 185 185 317 317 317 F F F
206042024-02-20T22:49:38.809Z 28 560-579 190 190 190 328 328 328 F F F
206052024-02-20T22:49:38.809Z 29 580-599 188 188 188 325 325 325 F F F
206062024-02-20T22:49:38.809ZMax gen: 191, Max flush: 331
206072024-02-20T22:49:38.809Z
206082024-02-20T22:49:38.809ZBegin loop 96
206092024-02-20T22:49:38.809ZDownstairs to restart: 2
206102024-02-20T22:49:38.820Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.32105073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906}
206112024-02-20T22:49:38.828Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
206122024-02-20T22:49:38.828Z,"time":"2024-02-20T22:49:38.321351599Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906}
206132024-02-20T22:49:38.828Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321385288Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906}
206142024-02-20T22:49:38.828Z{"msg":"Crucible 3b427f09-eb9f-4e27-b55c-269b37f2d833 has session id: b47f43f9-d627-44bc-8872-b341f82451c8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321422896Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206152024-02-20T22:49:38.828Z{"msg":"Upstairs opts: Upstairs UUID: 3b427f09-eb9f-4e27-b55c-269b37f2d833, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321481334Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206162024-02-20T22:49:38.828Z{"msg":"Crucible stats registered with UUID: 3b427f09-eb9f-4e27-b55c-269b37f2d833","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321550932Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206172024-02-20T22:49:38.828Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321581441Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906}
206182024-02-20T22:49:38.828Z{"msg":"3b427f09-eb9f-4e27-b55c-269b37f2d833 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.32161471Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206192024-02-20T22:49:38.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321631909Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206202024-02-20T22:49:38.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321658838Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206212024-02-20T22:49:38.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321685877Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206222024-02-20T22:49:38.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321716306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206232024-02-20T22:49:38.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321782044Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206242024-02-20T22:49:38.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.32186137Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206252024-02-20T22:49:38.828Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321938308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"io task","client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206262024-02-20T22:49:38.828Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.321988656Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"io task","client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206272024-02-20T22:49:38.828Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322033664Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"io task","client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206282024-02-20T22:49:38.828Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322057703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"io task","client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206292024-02-20T22:49:38.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322089693Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"io task","client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206302024-02-20T22:49:38.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322105952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"io task","client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206312024-02-20T22:49:38.829Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322644953Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206322024-02-20T22:49:38.829Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322696041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206332024-02-20T22:49:38.829Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.32272775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206342024-02-20T22:49:38.829Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322763548Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206352024-02-20T22:49:38.829Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322795407Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206362024-02-20T22:49:38.829Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.322824036Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206372024-02-20T22:49:38.829Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323314119Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206382024-02-20T22:49:38.829Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323356297Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206392024-02-20T22:49:38.829Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323387736Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206402024-02-20T22:49:38.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323845839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206412024-02-20T22:49:38.829Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323901018Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206422024-02-20T22:49:38.829Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323973065Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206432024-02-20T22:49:38.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.323989154Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206442024-02-20T22:49:38.829Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324003364Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206452024-02-20T22:49:38.829Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324025473Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206462024-02-20T22:49:38.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324052062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206472024-02-20T22:49:38.829Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324079001Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206482024-02-20T22:49:38.829Z{"msg":"[0]R flush_numbers[0..12]: [330, 331, 331, 328, 325, 331, 331, 317, 328, 331, 331, 331]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324099041Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206492024-02-20T22:49:38.829Z{"msg":"[0]R generation[0..12]: [190, 191, 191, 190, 188, 191, 191, 185, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.32412017Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206502024-02-20T22:49:38.829Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324135229Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206512024-02-20T22:49:38.829Z{"msg":"[1]R flush_numbers[0..12]: [330, 331, 331, 328, 325, 331, 331, 317, 328, 331, 331, 331]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324156508Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206522024-02-20T22:49:38.829Z{"msg":"[1]R generation[0..12]: [190, 191, 191, 190, 188, 191, 191, 185, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324170888Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206532024-02-20T22:49:38.829Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324191867Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206542024-02-20T22:49:38.830Z{"msg":"[2]R flush_numbers[0..12]: [330, 331, 331, 328, 325, 331, 331, 317, 328, 331, 331, 331]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324206327Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206552024-02-20T22:49:38.830Z{"msg":"[2]R generation[0..12]: [190, 191, 191, 190, 188, 191, 191, 185, 190, 191, 191, 191]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
206562024-02-20T22:49:38.830Z,"time":"2024-02-20T22:49:38.324228476Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206572024-02-20T22:49:38.830Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324247665Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206582024-02-20T22:49:38.830Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324273504Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206592024-02-20T22:49:38.830Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324287704Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206602024-02-20T22:49:38.830Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324307773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206612024-02-20T22:49:38.830Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324328602Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206622024-02-20T22:49:38.830Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324344672Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206632024-02-20T22:49:38.830Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324357131Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206642024-02-20T22:49:38.830Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324378Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206652024-02-20T22:49:38.830Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.3243916Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206662024-02-20T22:49:38.830Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324417839Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206672024-02-20T22:49:38.830Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324437468Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206682024-02-20T22:49:38.830Z{"msg":"3b427f09-eb9f-4e27-b55c-269b37f2d833 is now active with session: b47f43f9-d627-44bc-8872-b341f82451c8","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324472047Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206692024-02-20T22:49:38.830Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324506816Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906,"session_id":"b47f43f9-d627-44bc-8872-b341f82451c8"}
206702024-02-20T22:49:38.830Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:38.324522805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906}
206712024-02-20T22:49:38.830ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206722024-02-20T22:49:38.830ZLoading write count information from file "/tmp/test_repair_verify.data"
206732024-02-20T22:49:38.830ZRun Repair workload
206742024-02-20T22:49:38.830Z01/30 Write block 14 len 40960 data: 12 12 17 15 16 14 15 12 16 14
206752024-02-20T22:49:38.835Z02/30 Write block 347 len 16384 data: 13 15 13 15
206762024-02-20T22:49:38.835Z03/30 Write block 209 len 16384 data: 15 18 16 16
206772024-02-20T22:49:38.835Z04/30 Write block 373 len 28672 data: 17 14 13 14 19 16 15
206782024-02-20T22:49:38.835Z05/30 Write block 28 len 24576 data: 24 27 27 26 26 27
206792024-02-20T22:49:38.835Z06/30 Read block 558 len 28672
206802024-02-20T22:49:38.835Z07/30 Read block 316 len 36864
206812024-02-20T22:49:38.835Z08/30 Flush
206822024-02-20T22:49:38.835Z09/30 Write block 562 len 40960 data: 9 10 10 10 10 9 10 8 9 7
206832024-02-20T22:49:38.835Z10/30 Write block 441 len 36864 data: 17 16 13 12 15 14 14 12 10
206842024-02-20T22:49:38.835Z11/30 Read block 317 len 40960
206852024-02-20T22:49:38.840Z12/30 Read block 162 len 4096
206862024-02-20T22:49:38.840Z13/30 Read block 498 len 16384
206872024-02-20T22:49:38.840Z14/30 Flush
206882024-02-20T22:49:38.844Z15/30 Write block 547 len 4096 data: 15
206892024-02-20T22:49:38.844Z16/30 Flush
206902024-02-20T22:49:38.844Z17/30 Flush
206912024-02-20T22:49:38.844Z18/30 Write block 218 len 28672 data: 9 9 10 10 11 11 11
206922024-02-20T22:49:38.848Z19/30 Flush
206932024-02-20T22:49:38.848Z20/30 Write block 321 len 32768 data: 18 22 17 15 19 21 21 19
206942024-02-20T22:49:38.849Z21/30 Read block 587 len 32768
206952024-02-20T22:49:38.853Z22/30 Write block 21 len 12288 data: 13 17 15
206962024-02-20T22:49:38.853Z23/30 Read block 113 len 28672
206972024-02-20T22:49:38.853Z24/30 Read block 122 len 28672
206982024-02-20T22:49:38.853Z25/30 Read block 237 len 24576
206992024-02-20T22:49:38.853Z26/30 Write block 470 len 36864 data: 18 15 14 13 13 13 16 20 22
207002024-02-20T22:49:38.862Z27/30 Write block 287 len 24576 data: 16 15 12 12 12 9
207012024-02-20T22:49:38.862Z28/30 Read block 311 len 32768
207022024-02-20T22:49:38.862Z29/30 Write block 127 len 40960 data: 13 11 11 11 10 10 11 9 14 12
207032024-02-20T22:49:38.862Z30/30 Write block 480 len 12288 data: 21 21 21
207042024-02-20T22:49:38.862Z----------------------------------------------------------------
207052024-02-20T22:49:38.862Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207062024-02-20T22:49:38.862ZStates: Active Active Active
207072024-02-20T22:49:38.862ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207082024-02-20T22:49:38.862Z 1 Acked 1000 Write 10 Done Done Done false
207092024-02-20T22:49:38.862Z 2 Acked 1001 Write 4 Done Done Done false
207102024-02-20T22:49:38.862Z 3 Acked 1002 Write 4 Done Done Sent false
207112024-02-20T22:49:38.862Z 4 Acked 1003 Write 7 Done Done Sent false
207122024-02-20T22:49:38.862Z 5 Acked 1004 Write 6 Done Done Sent false
207132024-02-20T22:49:38.862Z 6 Acked 1005 Read 7 Done Done Sent false
207142024-02-20T22:49:38.862Z 7 Acked 1006 Read 9 Done Done Sent false
207152024-02-20T22:49:38.862Z 8 Acked 1007 Flush 0 Done Done Sent false
207162024-02-20T22:49:38.862Z 9 Acked 1008 Write 10 Done Done Sent false
207172024-02-20T22:49:38.862Z 10 Acked 1009 Write 9 Done Done Sent false
207182024-02-20T22:49:38.863Z 11 Acked 1010 Read 10 Done Done Sent false
207192024-02-20T22:49:38.863Z 12 Acked 1011 Read 1 Done Done Sent false
207202024-02-20T22:49:38.863Z 13 Acked 1012 Read 4 Done Done Sent false
207212024-02-20T22:49:38.863Z 14 Acked 1013 Flush 0 Done Done Sent false
207222024-02-20T22:49:38.863Z 15 Acked 1014 Write 1 Done Done Sent false
207232024-02-20T22:49:38.863Z 16 Acked 1015 Flush 0 Done Done Sent false
207242024-02-20T22:49:38.863Z 17 Acked 1016 Flush 0 Done Done Sent false
207252024-02-20T22:49:38.863Z 18 Acked 1017 Write 7 Done Done Sent false
207262024-02-20T22:49:38.863Z 19 Acked 1018 Flush 0 Done Done Sent false
207272024-02-20T22:49:38.863Z 20 Acked 1019 Write 8 Done Done Sent false
207282024-02-20T22:49:38.863Z 21 Acked 1020 Read 8 Done Done Sent false
207292024-02-20T22:49:38.863Z 22 Acked 1021 Write 3 Done Done Sent false
207302024-02-20T22:49:38.863Z 23 Acked 1022 Read 7 Done Done Sent false
207312024-02-20T22:49:38.863Z 24 Acked 1023 Read 7 Done Done Sent false
207322024-02-20T22:49:38.863Z 25 Acked 1024 Read 6 Done Done Sent false
207332024-02-20T22:49:38.863Z 26 Acked 1025 Write 9 Done Done Sent false
207342024-02-20T22:49:38.863Z 27 Acked 1026 Write 6 Done Done Sent false
207352024-02-20T22:49:38.863Z 28 Acked 1027 Read 8 Done Done Sent false
207362024-02-20T22:49:38.863Z 29 Acked 1028 Write 10 Sent Sent Sent false
207372024-02-20T22:49:38.863Z 30 Acked 1029 Write 3 Sent Sent Sent false
207382024-02-20T22:49:38.863Z STATES DS:0 DS:1 DS:2 TOTAL
207392024-02-20T22:49:38.863Z New 0 0 0 0
207402024-02-20T22:49:38.863Z Sent 2 2 28 32
207412024-02-20T22:49:38.863Z Done 28 28 2 58
207422024-02-20T22:49:38.863Z Skipped 0 0 0 0
207432024-02-20T22:49:38.863Z Error 0 0 0 0
207442024-02-20T22:49:38.864ZLast Flush: 1018 1018 0
207452024-02-20T22:49:38.864ZDownstairs last five completed:
207462024-02-20T22:49:38.864ZUpstairs last five completed: 30 29 28 27 26
207472024-02-20T22:49:38.864Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:38.358991512Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1906}
207482024-02-20T22:49:38.869ZWrote out file "/tmp/test_repair_verify.data"
207492024-02-20T22:49:38.869Z
207502024-02-20T22:49:38.869ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207512024-02-20T22:49:38.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207522024-02-20T22:49:38.886Z 0 000-019 192 192 190 332 332 330 F F T <---
207532024-02-20T22:49:38.886Z 1 020-039 192 192 191 332 332 331 T T T <---
207542024-02-20T22:49:38.886Z 2 040-059 191 191 191 331 331 331 F F F
207552024-02-20T22:49:38.886Z 3 060-079 190 190 190 328 328 328 F F F
207562024-02-20T22:49:38.886Z 4 080-099 188 188 188 325 325 325 F F F
207572024-02-20T22:49:38.886Z 5 100-119 191 191 191 331 331 331 F F F
207582024-02-20T22:49:38.886Z 6 120-139 191 191 191 331 331 331 T T F <---
207592024-02-20T22:49:38.886Z 7 140-159 185 185 185 317 317 317 F F F
207602024-02-20T22:49:38.887Z 8 160-179 190 190 190 328 328 328 F F F
207612024-02-20T22:49:38.887Z 9 180-199 191 191 191 331 331 331 F F F
207622024-02-20T22:49:38.887Z 10 200-219 192 192 191 336 336 331 F F F <---
207632024-02-20T22:49:38.887Z 11 220-239 192 192 191 336 336 331 F F F <---
207642024-02-20T22:49:38.887Z 12 240-259 188 188 188 325 325 325 F F F
207652024-02-20T22:49:38.897Z 13 260-279 188 188 188 325 325 325 F F F
207662024-02-20T22:49:38.897Z 14 280-299 191 191 191 331 331 331 T T F <---
207672024-02-20T22:49:38.897Z 15 300-319 191 191 191 331 331 331 F F F
207682024-02-20T22:49:38.897Z 16 320-339 191 191 191 331 331 331 T T F <---
207692024-02-20T22:49:38.897Z 17 340-359 192 192 180 332 332 307 F F T <---
207702024-02-20T22:49:38.897Z 18 360-379 192 192 191 332 332 331 F F F <---
207712024-02-20T22:49:38.897Z 19 380-399 190 190 190 330 330 330 F F F
207722024-02-20T22:49:38.897Z 20 400-419 186 186 186 322 322 322 F F F
207732024-02-20T22:49:38.897Z 21 420-439 190 190 190 328 328 328 F F F
207742024-02-20T22:49:38.897Z 22 440-459 192 192 186 333 333 319 F F F <---
207752024-02-20T22:49:38.897Z 23 460-479 188 188 188 324 324 324 T T F <---
207762024-02-20T22:49:38.898Z 24 480-499 190 190 190 330 330 330 T T F <---
207772024-02-20T22:49:38.898Z 25 500-519 190 190 190 330 330 330 F F F
207782024-02-20T22:49:38.898Z 26 520-539 190 190 190 328 328 328 F F F
207792024-02-20T22:49:38.898Z 27 540-559 192 192 185 334 334 317 F F F <---
207802024-02-20T22:49:38.898Z 28 560-579 192 192 190 333 333 328 F F F <---
207812024-02-20T22:49:38.898Z 29 580-599 188 188 188 325 325 325 F F F
207822024-02-20T22:49:38.898ZMax gen: 192, Max flush: 336
207832024-02-20T22:49:38.898ZError: Difference in extent metadata found!
207842024-02-20T22:49:38.898ZA Difference in extent metadata is expected here
207852024-02-20T22:49:38.898ZOn loop 96
207862024-02-20T22:49:38.898Z
207872024-02-20T22:49:38.898ZVerifying data now
207882024-02-20T22:49:43.592ZNo change after verify
207892024-02-20T22:49:43.592ZLoop: 96 Downstairs dump after verify (and repair):
207902024-02-20T22:49:43.613ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207912024-02-20T22:49:43.613Z 0 000-019 192 192 192 332 332 332 F F F
207922024-02-20T22:49:43.614Z 1 020-039 193 193 193 337 337 337 F F F
207932024-02-20T22:49:43.614Z 2 040-059 191 191 191 331 331 331 F F F
207942024-02-20T22:49:43.614Z 3 060-079 190 190 190 328 328 328 F F F
207952024-02-20T22:49:43.614Z 4 080-099 188 188 188 325 325 325 F F F
207962024-02-20T22:49:43.614Z 5 100-119 191 191 191 331 331 331 F F F
207972024-02-20T22:49:43.614Z 6 120-139 193 193 193 337 337 337 F F F
207982024-02-20T22:49:43.614Z 7 140-159 185 185 185 317 317 317 F F F
207992024-02-20T22:49:43.614Z 8 160-179 190 190 190 328 328 328 F F F
208002024-02-20T22:49:43.614Z 9 180-199 191 191 191 331 331 331 F F F
208012024-02-20T22:49:43.614Z 10 200-219 192 192 192 336 336 336 F F F
208022024-02-20T22:49:43.614Z 11 220-239 192 192 192 336 336 336 F F F
208032024-02-20T22:49:43.614Z 12 240-259 188 188 188 325 325 325 F F F
208042024-02-20T22:49:43.614Z 13 260-279 188 188 188 325 325 325 F F F
208052024-02-20T22:49:43.614Z 14 280-299 193 193 193 337 337 337 F F F
208062024-02-20T22:49:43.614Z 15 300-319 191 191 191 331 331 331 F F F
208072024-02-20T22:49:43.614Z 16 320-339 193 193 193 337 337 337 F F F
208082024-02-20T22:49:43.614Z 17 340-359 192 192 192 332 332 332 F F F
208092024-02-20T22:49:43.614Z 18 360-379 192 192 192 332 332 332 F F F
208102024-02-20T22:49:43.614Z 19 380-399 190 190 190 330 330 330 F F F
208112024-02-20T22:49:43.614Z 20 400-419 186 186 186 322 322 322 F F F
208122024-02-20T22:49:43.614Z 21 420-439 190 190 190 328 328 328 F F F
208132024-02-20T22:49:43.614Z 22 440-459 192 192 192 333 333 333 F F F
208142024-02-20T22:49:43.614Z 23 460-479 193 193 193 337 337 337 F F F
208152024-02-20T22:49:43.614Z 24 480-499 193 193 193 337 337 337 F F F
208162024-02-20T22:49:43.614Z 25 500-519 190 190 190 330 330 330 F F F
208172024-02-20T22:49:43.615Z 26 520-539 190 190 190 328 328 328 F F F
208182024-02-20T22:49:43.615Z 27 540-559 192 192 192 334 334 334 F F F
208192024-02-20T22:49:43.615Z 28 560-579 192 192 192 333 333 333 F F F
208202024-02-20T22:49:43.615Z 29 580-599 188 188 188 325 325 325 F F F
208212024-02-20T22:49:43.615ZMax gen: 193, Max flush: 337
208222024-02-20T22:49:43.615Z
208232024-02-20T22:49:43.615ZBegin loop 97
208242024-02-20T22:49:43.615ZDownstairs to restart: 2
208252024-02-20T22:49:43.643Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.118484834Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914}
208262024-02-20T22:49:43.643Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.118773073Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914}
208272024-02-20T22:49:43.643Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
208282024-02-20T22:49:43.643Z,"time":"2024-02-20T22:49:43.118810322Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914}
208292024-02-20T22:49:43.643Z{"msg":"Crucible a97d3504-e027-4344-bd60-6ebd01f1dd76 has session id: 98fa8e07-4c6d-4672-8def-5200723164a1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.11885703Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208302024-02-20T22:49:43.643Z{"msg":"Upstairs opts: Upstairs UUID: a97d3504-e027-4344-bd60-6ebd01f1dd76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.118907389Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208312024-02-20T22:49:43.643Z{"msg":"Crucible stats registered with UUID: a97d3504-e027-4344-bd60-6ebd01f1dd76","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.118943927Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208322024-02-20T22:49:43.643Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.118968306Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914}
208332024-02-20T22:49:43.643Z{"msg":"a97d3504-e027-4344-bd60-6ebd01f1dd76 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119043644Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208342024-02-20T22:49:43.644Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119071113Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208352024-02-20T22:49:43.644Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119117461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208362024-02-20T22:49:43.644Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.11915046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208372024-02-20T22:49:43.644Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119186219Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208382024-02-20T22:49:43.644Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119213038Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208392024-02-20T22:49:43.644Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119246966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208402024-02-20T22:49:43.644Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119275495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"io task","client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208412024-02-20T22:49:43.644Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119310664Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"io task","client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208422024-02-20T22:49:43.644Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119351683Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208432024-02-20T22:49:43.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119463529Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"io task","client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208442024-02-20T22:49:43.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119554125Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"io task","client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208452024-02-20T22:49:43.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.119634343Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208462024-02-20T22:49:43.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.12026828Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208472024-02-20T22:49:43.644Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.120297859Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208482024-02-20T22:49:43.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.120314128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208492024-02-20T22:49:43.644Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.120338857Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208502024-02-20T22:49:43.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.120353627Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208512024-02-20T22:49:43.644Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.120366966Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208522024-02-20T22:49:43.644Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.120961795Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208532024-02-20T22:49:43.644Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121001614Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208542024-02-20T22:49:43.644Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121079401Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208552024-02-20T22:49:43.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121567013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208562024-02-20T22:49:43.644Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121605582Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208572024-02-20T22:49:43.644Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121632101Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208582024-02-20T22:49:43.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.12166128Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208592024-02-20T22:49:43.645Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121678729Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208602024-02-20T22:49:43.645Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121700089Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208612024-02-20T22:49:43.645Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121713228Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208622024-02-20T22:49:43.645Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121734037Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208632024-02-20T22:49:43.645Z{"msg":"[0]R flush_numbers[0..12]: [332, 337, 331, 328, 325, 331, 337, 317, 328, 331, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121748537Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208642024-02-20T22:49:43.645Z{"msg":"[0]R generation[0..12]: [192, 193, 191, 190, 188, 191, 193, 185, 190, 191, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121769216Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208652024-02-20T22:49:43.645Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121784046Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208662024-02-20T22:49:43.645Z{"msg":"[1]R flush_numbers[0..12]: [332, 337, 331, 328, 325, 331, 337, 317, 328, 331, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121805715Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208672024-02-20T22:49:43.645Z{"msg":"[1]R generation[0..12]: [192, 193, 191, 190, 188, 191, 193, 185, 190, 191, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121820314Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208682024-02-20T22:49:43.645Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121841773Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208692024-02-20T22:49:43.645Z{"msg":"[2]R flush_numbers[0..12]: [332, 337, 331, 328, 325, 331, 337, 317, 328, 331, 336, 336]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
208702024-02-20T22:49:43.645Z2024-02-20T22:49:43.121856003Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208712024-02-20T22:49:43.645Z{"msg":"[2]R generation[0..12]: [192, 193, 191, 190, 188, 191, 193, 185, 190, 191, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121875822Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208722024-02-20T22:49:43.645Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121889782Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208732024-02-20T22:49:43.645Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121909771Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208742024-02-20T22:49:43.645Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121923271Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208752024-02-20T22:49:43.645Z{"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.1219461Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208762024-02-20T22:49:43.645Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121963159Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208772024-02-20T22:49:43.645Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.121983708Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208782024-02-20T22:49:43.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122019477Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208792024-02-20T22:49:43.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122044266Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208802024-02-20T22:49:43.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122058456Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208812024-02-20T22:49:43.645Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122072845Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208822024-02-20T22:49:43.645Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122092684Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208832024-02-20T22:49:43.646Z{"msg":"a97d3504-e027-4344-bd60-6ebd01f1dd76 is now active with session: 98fa8e07-4c6d-4672-8def-5200723164a1","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122105974Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208842024-02-20T22:49:43.646Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122118264Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914,"session_id":"98fa8e07-4c6d-4672-8def-5200723164a1"}
208852024-02-20T22:49:43.646Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:43.122137743Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914}
208862024-02-20T22:49:43.646ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208872024-02-20T22:49:43.646ZLoading write count information from file "/tmp/test_repair_verify.data"
208882024-02-20T22:49:43.646ZRun Repair workload
208892024-02-20T22:49:43.646Z01/30 Write block 264 len 40960 data: 11 13 13 13 12 12 10 12 12 10
208902024-02-20T22:49:43.646Z02/30 Read block 219 len 4096
208912024-02-20T22:49:43.664Z03/30 Read block 341 len 40960
208922024-02-20T22:49:43.664Z04/30 Read block 563 len 40960
208932024-02-20T22:49:43.664Z05/30 Read block 77 len 20480
208942024-02-20T22:49:43.664Z06/30 Read block 511 len 24576
208952024-02-20T22:49:43.664Z07/30 Write block 217 len 40960 data: 9 10 10 11 11 12 12 12 9 12
208962024-02-20T22:49:43.664Z08/30 Read block 361 len 8192
208972024-02-20T22:49:43.664Z09/30 Read block 204 len 40960
208982024-02-20T22:49:43.664Z10/30 Read block 138 len 16384
208992024-02-20T22:49:43.664Z11/30 Read block 443 len 16384
209002024-02-20T22:49:43.664Z12/30 Read block 539 len 12288
209012024-02-20T22:49:43.664Z13/30 Flush
209022024-02-20T22:49:43.664Z14/30 Write block 153 len 8192 data: 10 11
209032024-02-20T22:49:43.665Z15/30 Write block 70 len 24576 data: 17 17 17 13 12 10
209042024-02-20T22:49:43.665Z16/30 Read block 529 len 12288
209052024-02-20T22:49:43.665Z17/30 Flush
209062024-02-20T22:49:43.665Z18/30 Write block 461 len 16384 data: 23 22 23 19
209072024-02-20T22:49:43.665Z19/30 Read block 202 len 28672
209082024-02-20T22:49:43.665Z20/30 Read block 583 len 36864
209092024-02-20T22:49:43.665Z21/30 Flush
209102024-02-20T22:49:43.665Z22/30 Write block 379 len 36864 data: 16 16 13 16 15 15 13 12 11
209112024-02-20T22:49:43.665Z23/30 Read block 514 len 8192
209122024-02-20T22:49:43.665Z24/30 Read block 249 len 20480
209132024-02-20T22:49:43.665Z25/30 Flush
209142024-02-20T22:49:43.665Z26/30 Write block 565 len 8192 data: 11 11
209152024-02-20T22:49:43.665Z27/30 Write block 66 len 16384 data: 14 17 14 15
209162024-02-20T22:49:43.665Z28/30 Write block 124 len 16384 data: 18 14 11 14
209172024-02-20T22:49:43.665Z29/30 Write block 205 len 28672 data: 13 15 15 14 16 19 17
209182024-02-20T22:49:43.665Z30/30 Read block 247 len 28672
209192024-02-20T22:49:43.665Z----------------------------------------------------------------
209202024-02-20T22:49:43.665Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
209212024-02-20T22:49:43.665ZStates: Active Active Active
209222024-02-20T22:49:43.665ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209232024-02-20T22:49:43.665Z 1 Acked 1000 Write 10 Done Done Done false
209242024-02-20T22:49:43.665Z 2 Acked 1001 Read 1 Done Done Done false
209252024-02-20T22:49:43.665Z 3 Acked 1002 Read 10 Done Done Done false
209262024-02-20T22:49:43.665Z 4 Acked 1003 Read 10 Done Done Sent false
209272024-02-20T22:49:43.665Z 5 Acked 1004 Read 5 Done Done Sent false
209282024-02-20T22:49:43.666Z 6 Acked 1005 Read 6 Done Done Sent false
209292024-02-20T22:49:43.666Z 7 Acked 1006 Write 10 Done Done Sent false
209302024-02-20T22:49:43.666Z 8 Acked 1007 Read 2 Done Done Sent false
209312024-02-20T22:49:43.666Z 9 Acked 1008 Read 10 Done Done Sent false
209322024-02-20T22:49:43.666Z 10 Acked 1009 Read 4 Done Done Sent false
209332024-02-20T22:49:43.666Z 11 Acked 1010 Read 4 Done Done Sent false
209342024-02-20T22:49:43.666Z 12 Acked 1011 Read 3 Done Done Sent false
209352024-02-20T22:49:43.666Z 13 Acked 1012 Flush 0 Done Done Sent false
209362024-02-20T22:49:43.666Z 14 Acked 1013 Write 2 Done Done Sent false
209372024-02-20T22:49:43.666Z 15 Acked 1014 Write 6 Done Done Sent false
209382024-02-20T22:49:43.666Z 16 Acked 1015 Read 3 Done Done Sent false
209392024-02-20T22:49:43.666Z 17 Acked 1016 Flush 0 Done Done Sent false
209402024-02-20T22:49:43.666Z 18 Acked 1017 Write 4 Done Done Sent false
209412024-02-20T22:49:43.666Z 19 Acked 1018 Read 7 Done Done Sent false
209422024-02-20T22:49:43.666Z 20 Acked 1019 Read 9 Done Done Sent false
209432024-02-20T22:49:43.666Z 21 Acked 1020 Flush 0 Done Done Sent false
209442024-02-20T22:49:43.666Z 22 Acked 1021 Write 9 Done Done Sent false
209452024-02-20T22:49:43.666Z 23 Acked 1022 Read 2 Done Done Sent false
209462024-02-20T22:49:43.666Z 24 Acked 1023 Read 5 Done Done Sent false
209472024-02-20T22:49:43.666Z 25 Acked 1024 Flush 0 Done Done Sent false
209482024-02-20T22:49:43.666Z 26 Acked 1025 Write 2 Done Done Sent false
209492024-02-20T22:49:43.666Z 27 Acked 1026 Write 4 Done Done Sent false
209502024-02-20T22:49:43.666Z 28 Acked 1027 Write 4 Done Done Sent false
209512024-02-20T22:49:43.666Z 29 Acked 1028 Write 7 Done Done Sent false
209522024-02-20T22:49:43.666Z 30 Acked 1029 Read 7 Sent Done Sent false
209532024-02-20T22:49:43.667Z STATES DS:0 DS:1 DS:2 TOTAL
209542024-02-20T22:49:43.667Z New 0 0 0 0
209552024-02-20T22:49:43.667Z Sent 1 0 27 28
209562024-02-20T22:49:43.667Z Done 29 30 3 62
209572024-02-20T22:49:43.667Z Skipped 0 0 0 0
209582024-02-20T22:49:43.667Z Error 0 0 0 0
209592024-02-20T22:49:43.667ZLast Flush: 1024 1024 0
209602024-02-20T22:49:43.667ZDownstairs last five completed:
209612024-02-20T22:49:43.667ZUpstairs last five completed: 30 29 28 27 26
209622024-02-20T22:49:43.667Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:43.154317891Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1914}
209632024-02-20T22:49:43.667ZWrote out file "/tmp/test_repair_verify.data"
209642024-02-20T22:49:43.690Z
209652024-02-20T22:49:43.690ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209662024-02-20T22:49:43.690ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209672024-02-20T22:49:43.690Z 0 000-019 192 192 192 332 332 332 F F F
209682024-02-20T22:49:43.690Z 1 020-039 193 193 193 337 337 337 F F F
209692024-02-20T22:49:43.690Z 2 040-059 191 191 191 331 331 331 F F F
209702024-02-20T22:49:43.690Z 3 060-079 194 194 190 339 339 328 T T F <---
209712024-02-20T22:49:43.690Z 4 080-099 188 188 188 325 325 325 F F F
209722024-02-20T22:49:43.690Z 5 100-119 191 191 191 331 331 331 F F F
209732024-02-20T22:49:43.690Z 6 120-139 193 193 193 337 337 337 T T F <---
209742024-02-20T22:49:43.691Z 7 140-159 194 194 185 339 339 317 F F F <---
209752024-02-20T22:49:43.691Z 8 160-179 190 190 190 328 328 328 F F F
209762024-02-20T22:49:43.691Z 9 180-199 191 191 191 331 331 331 F F F
209772024-02-20T22:49:43.691Z 10 200-219 194 194 192 338 338 336 T T F <---
209782024-02-20T22:49:43.691Z 11 220-239 194 194 192 338 338 336 F F F <---
209792024-02-20T22:49:43.691Z 12 240-259 188 188 188 325 325 325 F F F
209802024-02-20T22:49:43.691Z 13 260-279 194 194 188 338 338 325 F F T <---
209812024-02-20T22:49:43.691Z 14 280-299 193 193 193 337 337 337 F F F
209822024-02-20T22:49:43.691Z 15 300-319 191 191 191 331 331 331 F F F
209832024-02-20T22:49:43.691Z 16 320-339 193 193 193 337 337 337 F F F
209842024-02-20T22:49:43.691Z 17 340-359 192 192 192 332 332 332 F F F
209852024-02-20T22:49:43.691Z 18 360-379 194 194 192 341 341 332 F F F <---
209862024-02-20T22:49:43.691Z 19 380-399 194 194 190 341 341 330 F F F <---
209872024-02-20T22:49:43.691Z 20 400-419 186 186 186 322 322 322 F F F
209882024-02-20T22:49:43.691Z 21 420-439 190 190 190 328 328 328 F F F
209892024-02-20T22:49:43.691Z 22 440-459 192 192 192 333 333 333 F F F
209902024-02-20T22:49:43.691Z 23 460-479 194 194 193 340 340 337 F F F <---
209912024-02-20T22:49:43.691Z 24 480-499 193 193 193 337 337 337 F F F
209922024-02-20T22:49:43.691Z 25 500-519 190 190 190 330 330 330 F F F
209932024-02-20T22:49:43.691Z 26 520-539 190 190 190 328 328 328 F F F
209942024-02-20T22:49:43.691Z 27 540-559 192 192 192 334 334 334 F F F
209952024-02-20T22:49:43.691Z 28 560-579 192 192 192 333 333 333 T T F <---
209962024-02-20T22:49:43.691Z 29 580-599 188 188 188 325 325 325 F F F
209972024-02-20T22:49:43.691ZMax gen: 194, Max flush: 341
209982024-02-20T22:49:43.691ZError: Difference in extent metadata found!
209992024-02-20T22:49:43.703ZA Difference in extent metadata is expected here
210002024-02-20T22:49:43.703ZOn loop 97
210012024-02-20T22:49:43.703Z
210022024-02-20T22:49:43.703ZVerifying data now
210032024-02-20T22:49:48.192ZNo change after verify
210042024-02-20T22:49:48.192ZLoop: 97 Downstairs dump after verify (and repair):
210052024-02-20T22:49:48.203ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210062024-02-20T22:49:48.204Z 0 000-019 192 192 192 332 332 332 F F F
210072024-02-20T22:49:48.204Z 1 020-039 193 193 193 337 337 337 F F F
210082024-02-20T22:49:48.204Z 2 040-059 191 191 191 331 331 331 F F F
210092024-02-20T22:49:48.204Z 3 060-079 195 195 195 342 342 342 F F F
210102024-02-20T22:49:48.204Z 4 080-099 188 188 188 325 325 325 F F F
210112024-02-20T22:49:48.204Z 5 100-119 191 191 191 331 331 331 F F F
210122024-02-20T22:49:48.205Z 6 120-139 195 195 195 342 342 342 F F F
210132024-02-20T22:49:48.205Z 7 140-159 194 194 194 339 339 339 F F F
210142024-02-20T22:49:48.205Z 8 160-179 190 190 190 328 328 328 F F F
210152024-02-20T22:49:48.205Z 9 180-199 191 191 191 331 331 331 F F F
210162024-02-20T22:49:48.205Z 10 200-219 195 195 195 342 342 342 F F F
210172024-02-20T22:49:48.210Z 11 220-239 194 194 194 338 338 338 F F F
210182024-02-20T22:49:48.210Z 12 240-259 188 188 188 325 325 325 F F F
210192024-02-20T22:49:48.210Z 13 260-279 194 194 194 338 338 338 F F F
210202024-02-20T22:49:48.210Z 14 280-299 193 193 193 337 337 337 F F F
210212024-02-20T22:49:48.210Z 15 300-319 191 191 191 331 331 331 F F F
210222024-02-20T22:49:48.210Z 16 320-339 193 193 193 337 337 337 F F F
210232024-02-20T22:49:48.210Z 17 340-359 192 192 192 332 332 332 F F F
210242024-02-20T22:49:48.210Z 18 360-379 194 194 194 341 341 341 F F F
210252024-02-20T22:49:48.210Z 19 380-399 194 194 194 341 341 341 F F F
210262024-02-20T22:49:48.210Z 20 400-419 186 186 186 322 322 322 F F F
210272024-02-20T22:49:48.210Z 21 420-439 190 190 190 328 328 328 F F F
210282024-02-20T22:49:48.210Z 22 440-459 192 192 192 333 333 333 F F F
210292024-02-20T22:49:48.210Z 23 460-479 194 194 194 340 340 340 F F F
210302024-02-20T22:49:48.210Z 24 480-499 193 193 193 337 337 337 F F F
210312024-02-20T22:49:48.210Z 25 500-519 190 190 190 330 330 330 F F F
210322024-02-20T22:49:48.211Z 26 520-539 190 190 190 328 328 328 F F F
210332024-02-20T22:49:48.211Z 27 540-559 192 192 192 334 334 334 F F F
210342024-02-20T22:49:48.211Z 28 560-579 195 195 195 342 342 342 F F F
210352024-02-20T22:49:48.211Z 29 580-599 188 188 188 325 325 325 F F F
210362024-02-20T22:49:48.211ZMax gen: 195, Max flush: 342
210372024-02-20T22:49:48.211Z
210382024-02-20T22:49:48.211ZBegin loop 98
210392024-02-20T22:49:48.211ZDownstairs to restart: 2
210402024-02-20T22:49:48.227Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.726662399Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922}
210412024-02-20T22:49:48.249Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727010057Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922}
210422024-02-20T22:49:48.250Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727055095Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922}
210432024-02-20T22:49:48.250Z{"msg":"Crucible 7a318353-8fb6-4a6a-a11f-3c13ffe62d3b has session id: bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727076114Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922Crucible runtime is spawned
210442024-02-20T22:49:48.250Z,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210452024-02-20T22:49:48.250Z{"msg":"Upstairs opts: Upstairs UUID: 7a318353-8fb6-4a6a-a11f-3c13ffe62d3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727134692Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210462024-02-20T22:49:48.250Z{"msg":"Crucible stats registered with UUID: 7a318353-8fb6-4a6a-a11f-3c13ffe62d3b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727153591Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210472024-02-20T22:49:48.250Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.72718296Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922}
210482024-02-20T22:49:48.250Z{"msg":"7a318353-8fb6-4a6a-a11f-3c13ffe62d3b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727306636Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210492024-02-20T22:49:48.250Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727338585Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210502024-02-20T22:49:48.250Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727357874Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210512024-02-20T22:49:48.250Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727406522Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210522024-02-20T22:49:48.250Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727450071Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210532024-02-20T22:49:48.251Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727548737Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210542024-02-20T22:49:48.251Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727590316Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210552024-02-20T22:49:48.251Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727617225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"io task","client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210562024-02-20T22:49:48.251Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727767829Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"io task","client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210572024-02-20T22:49:48.251Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727807928Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"io task","client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210582024-02-20T22:49:48.251Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727825487Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"io task","client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210592024-02-20T22:49:48.251Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727852866Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"io task","client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210602024-02-20T22:49:48.251Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.727867956Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"io task","client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210612024-02-20T22:49:48.251Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.72831971Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210622024-02-20T22:49:48.251Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.728355168Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210632024-02-20T22:49:48.251Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.728370778Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210642024-02-20T22:49:48.251Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.728384447Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210652024-02-20T22:49:48.251Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.728411666Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210662024-02-20T22:49:48.252Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.728425996Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210672024-02-20T22:49:48.252Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.728973176Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210682024-02-20T22:49:48.252Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729008775Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210692024-02-20T22:49:48.252Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729024425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210702024-02-20T22:49:48.252Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.72942847Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210712024-02-20T22:49:48.252Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729514717Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210722024-02-20T22:49:48.252Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729569395Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210732024-02-20T22:49:48.252Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729605534Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210742024-02-20T22:49:48.252Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729631533Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210752024-02-20T22:49:48.252Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729660062Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210762024-02-20T22:49:48.252Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729682351Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210772024-02-20T22:49:48.252Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.72971077Z","hostname":Wait for a query_work_queue command to finish before sending IO
210782024-02-20T22:49:48.252Z"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210792024-02-20T22:49:48.253Z{"msg":"[0]R flush_numbers[0..12]: [332, 337, 331, 342, 325, 331, 342, 339, 328, 331, 342, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729741139Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210802024-02-20T22:49:48.253Z{"msg":"[0]R generation[0..12]: [192, 193, 191, 195, 188, 191, 195, 194, 190, 191, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729774418Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210812024-02-20T22:49:48.253Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729801067Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210822024-02-20T22:49:48.253Z{"msg":"[1]R flush_numbers[0..12]: [332, 337, 331, 342, 325, 331, 342, 339, 328, 331, 342, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729834256Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210832024-02-20T22:49:48.253Z{"msg":"[1]R generation[0..12]: [192, 193, 191, 195, 188, 191, 195, 194, 190, 191, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729860495Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210842024-02-20T22:49:48.253Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729893753Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210852024-02-20T22:49:48.253Z{"msg":"[2]R flush_numbers[0..12]: [332, 337, 331, 342, 325, 331, 342, 339, 328, 331, 342, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729919892Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210862024-02-20T22:49:48.253Z{"msg":"[2]R generation[0..12]: [192, 193, 191, 195, 188, 191, 195, 194, 190, 191, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.729951851Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210872024-02-20T22:49:48.253Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.72997799Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210882024-02-20T22:49:48.253Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730009679Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210892024-02-20T22:49:48.253Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730034398Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pidRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210902024-02-20T22:49:48.253Z":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210912024-02-20T22:49:48.253Z{"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730069637Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210922024-02-20T22:49:48.254Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730093586Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210932024-02-20T22:49:48.254Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730116935Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210942024-02-20T22:49:48.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730137805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210952024-02-20T22:49:48.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730168744Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210962024-02-20T22:49:48.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730192533Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210972024-02-20T22:49:48.254Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730219202Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210982024-02-20T22:49:48.254Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730241211Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
210992024-02-20T22:49:48.254Z{"msg":"7a318353-8fb6-4a6a-a11f-3c13ffe62d3b is now active with session: bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.73026869Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
211002024-02-20T22:49:48.254Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730290249Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922,"session_id":"bb5ad48b-5fc9-4f4a-8ca9-ef2c5146a717"}
211012024-02-20T22:49:48.254Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:47.730316268Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922}
211022024-02-20T22:49:48.254ZLoading write count information from file "/tmp/test_repair_verify.data"
211032024-02-20T22:49:48.254ZRun Repair workload
211042024-02-20T22:49:48.254Z01/30 Write block 534 len 4096 data: 19
211052024-02-20T22:49:48.254Z02/30 Read block 497 len 36864
211062024-02-20T22:49:48.260Z03/30 Write block 33 len 12288 data: 28 25 26
211072024-02-20T22:49:48.260Z04/30 Write block 414 len 16384 data: 24 23 19 17
211082024-02-20T22:49:48.260Z05/30 Flush
211092024-02-20T22:49:48.260Z06/30 Write block 452 len 36864 data: 13 11 13 14 22 22 25 22 20
211102024-02-20T22:49:48.260Z07/30 Flush
211112024-02-20T22:49:48.260Z08/30 Write block 539 len 8192 data: 18 17
211122024-02-20T22:49:48.260Z09/30 Write block 531 len 32768 data: 20 16 18 20 22 20 18 23
211132024-02-20T22:49:48.260Z10/30 Write block 329 len 36864 data: 17 15 14 19 21 18 15 16 16
211142024-02-20T22:49:48.260Z11/30 Write block 538 len 20480 data: 24 19 18 18 14
211152024-02-20T22:49:48.260Z12/30 Read block 441 len 32768
211162024-02-20T22:49:48.261Z13/30 Read block 567 len 32768
211172024-02-20T22:49:48.261Z14/30 Write block 112 len 20480 data: 16 17 16 15 18
211182024-02-20T22:49:48.261Z15/30 Write block 362 len 28672 data: 23 24 24 24 23 16 18
211192024-02-20T22:49:48.261Z16/30 Read block 438 len 20480
211202024-02-20T22:49:48.261Z17/30 Read block 162 len 4096
211212024-02-20T22:49:48.261Z18/30 Write block 379 len 12288 data: 17 17 14
211222024-02-20T22:49:48.261Z19/30 Read block 582 len 4096
211232024-02-20T22:49:48.261Z20/30 Write block 178 len 8192 data: 16 20
211242024-02-20T22:49:48.261Z21/30 Write block 257 len 36864 data: 10 11 12 11 10 8 7 12 14
211252024-02-20T22:49:48.261Z22/30 Flush
211262024-02-20T22:49:48.261Z23/30 Write block 405 len 36864 data: 17 16 16 14 18 19 19 21 24
211272024-02-20T22:49:48.261Z24/30 Write block 296 len 24576 data: 11 13 17 17 14 19
211282024-02-20T22:49:48.261Z25/30 Read block 336 len 32768
211292024-02-20T22:49:48.261Z26/30 Write block 79 len 28672 data: 9 11 8 8 8 8 8
211302024-02-20T22:49:48.262Z27/30 Read block 455 len 8192
211312024-02-20T22:49:48.267Z28/30 Write block 283 len 24576 data: 13 13 16 15 17 16
211322024-02-20T22:49:48.267Z29/30 Read block 255 len 24576
211332024-02-20T22:49:48.267Z30/30 Write block 591 len 20480 data: 15 14 14 15 15
211342024-02-20T22:49:48.267Z----------------------------------------------------------------
211352024-02-20T22:49:48.267Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
211362024-02-20T22:49:48.267ZStates: Active Active Active
211372024-02-20T22:49:48.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211382024-02-20T22:49:48.268Z 1 Acked 1000 Write 1 Done Done Sent false
211392024-02-20T22:49:48.268Z 2 Acked 1001 Read 9 Done Done Sent false
211402024-02-20T22:49:48.268Z 3 Acked 1002 Write 3 Done Done Sent false
211412024-02-20T22:49:48.268Z 4 Acked 1003 Write 4 Done Done Sent false
211422024-02-20T22:49:48.268Z 5 Acked 1004 Flush 0 Done Done Sent false
211432024-02-20T22:49:48.268Z 6 Acked 1005 Write 9 Done Done Sent false
211442024-02-20T22:49:48.268Z 7 Acked 1006 Flush 0 Done Done Sent false
211452024-02-20T22:49:48.268Z 8 Acked 1007 Write 2 Done Done Sent false
211462024-02-20T22:49:48.269Z 9 Acked 1008 Write 8 Done Done Sent false
211472024-02-20T22:49:48.269Z 10 Acked 1009 Write 9 Done Done Sent false
211482024-02-20T22:49:48.269Z 11 Acked 1010 Write 5 Done Done Sent false
211492024-02-20T22:49:48.269Z 12 Acked 1011 Read 8 Done Done Sent false
211502024-02-20T22:49:48.269Z 13 Acked 1012 Read 8 Done Done Sent false
211512024-02-20T22:49:48.269Z 14 Acked 1013 Write 5 Done Done Sent false
211522024-02-20T22:49:48.269Z 15 Acked 1014 Write 7 Done Done Sent false
211532024-02-20T22:49:48.269Z 16 Acked 1015 Read 5 Done Done Sent false
211542024-02-20T22:49:48.269Z 17 Acked 1016 Read 1 Done Done Sent false
211552024-02-20T22:49:48.269Z 18 Acked 1017 Write 3 Done Done Sent false
211562024-02-20T22:49:48.269Z 19 Acked 1018 Read 1 Done Done Sent false
211572024-02-20T22:49:48.269Z 20 Acked 1019 Write 2 Done Done Sent false
211582024-02-20T22:49:48.269Z 21 Acked 1020 Write 9 Done Done Sent false
211592024-02-20T22:49:48.269Z 22 Acked 1021 Flush 0 Done Done Sent false
211602024-02-20T22:49:48.270Z 23 Acked 1022 Write 9 Done Done Sent false
211612024-02-20T22:49:48.270Z 24 Acked 1023 Write 6 Done Done Sent false
211622024-02-20T22:49:48.270Z 25 Acked 1024 Read 8 Done Done Sent false
211632024-02-20T22:49:48.270Z 26 Acked 1025 Write 7 Done Done Sent false
211642024-02-20T22:49:48.270Z 27 Acked 1026 Read 2 Done Done Sent false
211652024-02-20T22:49:48.270Z 28 Acked 1027 Write 6 Done Done Sent false
211662024-02-20T22:49:48.270Z 29 Acked 1028 Read 6 Done Done Sent false
211672024-02-20T22:49:48.270Z 30 Acked 1029 Write 5 Sent Sent Sent false
211682024-02-20T22:49:48.270Z STATES DS:0 DS:1 DS:2 TOTAL
211692024-02-20T22:49:48.270Z New 0 0 0 0
211702024-02-20T22:49:48.270Z Sent 1 1 30 32
211712024-02-20T22:49:48.270Z Done 29 29 0 58
211722024-02-20T22:49:48.270Z Skipped 0 0 0 0
211732024-02-20T22:49:48.270Z Error 0 0 0 0
211742024-02-20T22:49:48.271ZLast Flush: 1021 1021 0
211752024-02-20T22:49:48.271ZDownstairs last five completed:
211762024-02-20T22:49:48.271ZUpstairs last five completed: 30 29 28 27 26
211772024-02-20T22:49:48.271Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:47.763941225Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1922}
211782024-02-20T22:49:48.271ZWrote out file "/tmp/test_repair_verify.data"
211792024-02-20T22:49:48.276Z
211802024-02-20T22:49:48.276ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211812024-02-20T22:49:48.306ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211822024-02-20T22:49:48.306Z 0 000-019 192 192 192 332 332 332 F F F
211832024-02-20T22:49:48.306Z 1 020-039 196 196 193 343 343 337 F F F <---
211842024-02-20T22:49:48.306Z 2 040-059 191 191 191 331 331 331 F F F
211852024-02-20T22:49:48.306Z 3 060-079 195 195 195 342 342 342 T T F <---
211862024-02-20T22:49:48.306Z 4 080-099 188 188 188 325 325 325 T T F <---
211872024-02-20T22:49:48.306Z 5 100-119 196 196 191 345 345 331 F F F <---
211882024-02-20T22:49:48.306Z 6 120-139 195 195 195 342 342 342 F F F
211892024-02-20T22:49:48.329Z 7 140-159 194 194 194 339 339 339 F F F
211902024-02-20T22:49:48.329Z 8 160-179 196 196 190 345 345 328 F F F <---
211912024-02-20T22:49:48.329Z 9 180-199 191 191 191 331 331 331 F F F
211922024-02-20T22:49:48.329Z 10 200-219 195 195 195 342 342 342 F F F
211932024-02-20T22:49:48.329Z 11 220-239 194 194 194 338 338 338 F F F
211942024-02-20T22:49:48.329Z 12 240-259 196 196 188 345 345 325 F F F <---
211952024-02-20T22:49:48.329Z 13 260-279 196 196 194 345 345 338 F F F <---
211962024-02-20T22:49:48.329Z 14 280-299 193 193 193 337 337 337 T T F <---
211972024-02-20T22:49:48.329Z 15 300-319 191 191 191 331 331 331 T T F <---
211982024-02-20T22:49:48.329Z 16 320-339 196 196 193 345 345 337 F F F <---
211992024-02-20T22:49:48.329Z 17 340-359 192 192 192 332 332 332 F F F
212002024-02-20T22:49:48.329Z 18 360-379 196 196 194 345 345 341 F F F <---
212012024-02-20T22:49:48.329Z 19 380-399 196 196 194 345 345 341 F F F <---
212022024-02-20T22:49:48.329Z 20 400-419 196 196 186 343 343 322 T T F <---
212032024-02-20T22:49:48.329Z 21 420-439 190 190 190 328 328 328 F F F
212042024-02-20T22:49:48.329Z 22 440-459 196 196 192 344 344 333 F F F <---
212052024-02-20T22:49:48.329Z 23 460-479 196 196 194 344 344 340 F F F <---
212062024-02-20T22:49:48.329Z 24 480-499 193 193 193 337 337 337 F F F
212072024-02-20T22:49:48.329Z 25 500-519 190 190 190 330 330 330 F F F
212082024-02-20T22:49:48.329Z 26 520-539 196 196 190 345 345 328 F F F <---
212092024-02-20T22:49:48.330Z 27 540-559 196 196 192 345 345 334 F F F <---
212102024-02-20T22:49:48.330Z 28 560-579 195 195 195 342 342 342 F F F
212112024-02-20T22:49:48.330Z 29 580-599 188 188 188 325 325 325 T T F <---
212122024-02-20T22:49:48.330ZMax gen: 196, Max flush: 345
212132024-02-20T22:49:48.330ZError: Difference in extent metadata found!
212142024-02-20T22:49:48.330ZA Difference in extent metadata is expected here
212152024-02-20T22:49:48.330ZOn loop 98
212162024-02-20T22:49:48.330Z
212172024-02-20T22:49:48.330ZVerifying data now
212182024-02-20T22:49:53.147ZNo change after verify
212192024-02-20T22:49:53.148ZLoop: 98 Downstairs dump after verify (and repair):
212202024-02-20T22:49:53.575ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212212024-02-20T22:49:53.575Z 0 000-019 192 192 192 332 332 332 F F F
212222024-02-20T22:49:53.576Z 1 020-039 196 196 196 343 343 343 F F F
212232024-02-20T22:49:53.576Z 2 040-059 191 191 191 331 331 331 F F F
212242024-02-20T22:49:53.576Z 3 060-079 197 197 197 346 346 346 F F F
212252024-02-20T22:49:53.576Z 4 080-099 197 197 197 346 346 346 F F F
212262024-02-20T22:49:53.576Z 5 100-119 196 196 196 345 345 345 F F F
212272024-02-20T22:49:53.577Z 6 120-139 195 195 195 342 342 342 F F F
212282024-02-20T22:49:53.577Z 7 140-159 194 194 194 339 339 339 F F F
212292024-02-20T22:49:53.577Z 8 160-179 196 196 196 345 345 345 F F F
212302024-02-20T22:49:53.577Z 9 180-199 191 191 191 331 331 331 F F F
212312024-02-20T22:49:53.577Z 10 200-219 195 195 195 342 342 342 F F F
212322024-02-20T22:49:53.577Z 11 220-239 194 194 194 338 338 338 F F F
212332024-02-20T22:49:53.578Z 12 240-259 196 196 196 345 345 345 F F F
212342024-02-20T22:49:53.578Z 13 260-279 196 196 196 345 345 345 F F F
212352024-02-20T22:49:53.578Z 14 280-299 197 197 197 346 346 346 F F F
212362024-02-20T22:49:53.578Z 15 300-319 197 197 197 346 346 346 F F F
212372024-02-20T22:49:53.578Z 16 320-339 196 196 196 345 345 345 F F F
212382024-02-20T22:49:53.578Z 17 340-359 192 192 192 332 332 332 F F F
212392024-02-20T22:49:53.579Z 18 360-379 196 196 196 345 345 345 F F F
212402024-02-20T22:49:53.579Z 19 380-399 196 196 196 345 345 345 F F F
212412024-02-20T22:49:53.579Z 20 400-419 197 197 197 346 346 346 F F F
212422024-02-20T22:49:53.579Z 21 420-439 190 190 190 328 328 328 F F F
212432024-02-20T22:49:53.579Z 22 440-459 196 196 196 344 344 344 F F F
212442024-02-20T22:49:53.579Z 23 460-479 196 196 196 344 344 344 F F F
212452024-02-20T22:49:53.579Z 24 480-499 193 193 193 337 337 337 F F F
212462024-02-20T22:49:53.579Z 25 500-519 190 190 190 330 330 330 F F F
212472024-02-20T22:49:53.580Z 26 520-539 196 196 196 345 345 345 F F F
212482024-02-20T22:49:53.580Z 27 540-559 196 196 196 345 345 345 F F F
212492024-02-20T22:49:53.580Z 28 560-579 195 195 195 342 342 342 F F F
212502024-02-20T22:49:53.580Z 29 580-599 197 197 197 346 346 346 F F F
212512024-02-20T22:49:53.580ZMax gen: 197, Max flush: 346
212522024-02-20T22:49:53.580Z
212532024-02-20T22:49:53.581ZBegin loop 99
212542024-02-20T22:49:53.581ZDownstairs to restart: 1
212552024-02-20T22:49:53.581Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.672534981Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930}
212562024-02-20T22:49:53.581Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4165a93f49e1788224227a01e70716208ae85396\",\n git_commit_timestamp: \"2024-02-20T22:21:31.000000000Z\",\n git_branch: \"alan/panic-from-the-top\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.67283716Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930}
212572024-02-20T22:49:53.581Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":Crucible runtime is spawned
212582024-02-20T22:49:53.581Z30,"time":"2024-02-20T22:49:52.672878499Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930}
212592024-02-20T22:49:53.582Z{"msg":"Crucible 8a26cc5c-bf2b-4492-bf64-a424736cbb4b has session id: d8d72a3a-f7e6-44ed-9e03-e65501cce6c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.672920767Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212602024-02-20T22:49:53.582Z{"msg":"Upstairs opts: Upstairs UUID: 8a26cc5c-bf2b-4492-bf64-a424736cbb4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.672968425Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212612024-02-20T22:49:53.582Z{"msg":"Crucible stats registered with UUID: 8a26cc5c-bf2b-4492-bf64-a424736cbb4b","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673009634Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212622024-02-20T22:49:53.582Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673027903Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930}
212632024-02-20T22:49:53.583Z{"msg":"8a26cc5c-bf2b-4492-bf64-a424736cbb4b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673232016Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212642024-02-20T22:49:53.583Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673264865Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212652024-02-20T22:49:53.583Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673283894Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212662024-02-20T22:49:53.583Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673309543Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212672024-02-20T22:49:53.583Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673327453Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212682024-02-20T22:49:53.583Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673345252Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212692024-02-20T22:49:53.584Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673370381Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212702024-02-20T22:49:53.584Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673414689Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"io task","client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212712024-02-20T22:49:53.584Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673464098Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"io task","client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212722024-02-20T22:49:53.584Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673512406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"io task","client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212732024-02-20T22:49:53.584Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673552415Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"io task","client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212742024-02-20T22:49:53.584Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673581723Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"io task","client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212752024-02-20T22:49:53.584Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.673615952Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"io task","client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212762024-02-20T22:49:53.584Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.674310157Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212772024-02-20T22:49:53.585Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.674348786Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212782024-02-20T22:49:53.585Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.674384805Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212792024-02-20T22:49:53.585Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.674400304Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212802024-02-20T22:49:53.585Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.674498781Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212812024-02-20T22:49:53.585Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.674566618Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212822024-02-20T22:49:53.585Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675014762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212832024-02-20T22:49:53.585Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675159557Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212842024-02-20T22:49:53.585Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675298212Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212852024-02-20T22:49:53.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675566383Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212862024-02-20T22:49:53.586Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.67563546Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212872024-02-20T22:49:53.586Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675676308Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212882024-02-20T22:49:53.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675817503Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212892024-02-20T22:49:53.586Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675864082Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212902024-02-20T22:49:53.586Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675881731Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212912024-02-20T22:49:53.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.67590945Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212922024-02-20T22:49:53.586Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.67592493Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212932024-02-20T22:49:53.586Z{"msg":"[0]R flush_numbers[0..12]: [332, 343, 331, 346, 346, 345, 342, 339, 345, 331, 342, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675954829Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212942024-02-20T22:49:53.586Z{"msg":"[0]R generation[0..12]: [192, 196, 191, 197, 197, 196, 195, 194, 196, 191, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675977968Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212952024-02-20T22:49:53.586Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.675992887Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212962024-02-20T22:49:53.587Z{"msg":"[1]R flush_numbers[0..12]: [332, 343, 331, 346, 346, 345, 342, 339, 345, 331, 342, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676030396Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212972024-02-20T22:49:53.587Z{"msg":"[1]R generation[0..12]: [192, 196, 191, 197, 197, 196, 195, 194, 196, 191, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676045555Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212982024-02-20T22:49:53.587Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676067034Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
212992024-02-20T22:49:53.587Z{"msg":"[2]R flush_numbers[0..12]: [332, 343, 331, 346, 346, 345, 342, 339, 345, 331, 342, 338]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
213002024-02-20T22:49:53.587Z2024-02-20T22:49:52.676081434Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213012024-02-20T22:49:53.587Z{"msg":"[2]R generation[0..12]: [192, 196, 191, 197, 197, 196, 195, 194, 196, 191, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676112373Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213022024-02-20T22:49:53.587Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676133262Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213032024-02-20T22:49:53.587Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676146762Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213042024-02-20T22:49:53.587Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676166821Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213052024-02-20T22:49:53.587Z{"msg":"Next flush: 347","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.67618013Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213062024-02-20T22:49:53.588Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.67619349Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213072024-02-20T22:49:53.588Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676213469Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213082024-02-20T22:49:53.588Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676226179Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213092024-02-20T22:49:53.588Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676246078Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213102024-02-20T22:49:53.588Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676259798Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213112024-02-20T22:49:53.588Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676279617Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213122024-02-20T22:49:53.588Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676292406Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213132024-02-20T22:49:53.588Z{"msg":"8a26cc5c-bf2b-4492-bf64-a424736cbb4b is now active with session: d8d72a3a-f7e6-44ed-9e03-e65501cce6c9","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676312076Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213142024-02-20T22:49:53.589Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676324485Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930,"session_id":"d8d72a3a-f7e6-44ed-9e03-e65501cce6c9"}
213152024-02-20T22:49:53.589Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-20T22:49:52.676343255Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930}
213162024-02-20T22:49:53.589ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213172024-02-20T22:49:53.589ZLoading write count information from file "/tmp/test_repair_verify.data"
213182024-02-20T22:49:53.589ZRun Repair workload
213192024-02-20T22:49:53.589Z01/30 Write block 500 len 12288 data: 9 8 9
213202024-02-20T22:49:53.600Z02/30 Read block 435 len 36864
213212024-02-20T22:49:53.600Z03/30 Read block 109 len 4096
213222024-02-20T22:49:53.600Z04/30 Write block 227 len 40960 data: 13 16 19 23 20 22 24 24 21 21
213232024-02-20T22:49:53.600Z05/30 Read block 5 len 32768
213242024-02-20T22:49:53.600Z06/30 Flush
213252024-02-20T22:49:53.600Z07/30 Write block 7 len 24576 data: 8 7 10 13 11 10
213262024-02-20T22:49:53.600Z08/30 Read block 198 len 20480
213272024-02-20T22:49:53.601Z09/30 Read block 463 len 24576
213282024-02-20T22:49:53.601Z10/30 Write block 39 len 12288 data: 20 15 15
213292024-02-20T22:49:53.601Z11/30 Write block 180 len 24576 data: 18 22 20 16 18 15
213302024-02-20T22:49:53.601Z12/30 Write block 447 len 4096 data: 15
213312024-02-20T22:49:53.601Z13/30 Read block 442 len 28672
213322024-02-20T22:49:53.601Z14/30 Read block 402 len 20480
213332024-02-20T22:49:53.601Z15/30 Write block 202 len 24576 data: 10 12 12 14 16 16
213342024-02-20T22:49:53.601Z16/30 Write block 27 len 12288 data: 24 25 28
213352024-02-20T22:49:53.602Z17/30 Write block 426 len 8192 data: 13 12
213362024-02-20T22:49:53.602Z18/30 Write block 81 len 40960 data: 9 9 9 9 9 7 9 11 11 13
213372024-02-20T22:49:53.602Z19/30 Read block 275 len 40960
213382024-02-20T22:49:53.602Z20/30 Write block 27 len 8192 data: 25 26
213392024-02-20T22:49:53.602Z21/30 Read block 449 len 12288
213402024-02-20T22:49:53.602Z22/30 Read block 449 len 28672
213412024-02-20T22:49:53.602Z23/30 Write block 307 len 24576 data: 17 15 14 13 13 12
213422024-02-20T22:49:53.603Z24/30 Write block 189 len 24576 data: 14 13 14 15 14 13
213432024-02-20T22:49:53.603Z25/30 Read block 254 len 24576
213442024-02-20T22:49:53.603Z26/30 Write block 163 len 24576 data: 11 11 14 14 12 12
213452024-02-20T22:49:53.603Z27/30 Read block 16 len 8192
213462024-02-20T22:49:53.603Z28/30 Write block 202 len 32768 data: 11 13 13 15 17 17 15 17
213472024-02-20T22:49:53.603Z29/30 Read block 337 len 24576
213482024-02-20T22:49:53.603Z30/30 Write block 81 len 8192 data: 10 10
213492024-02-20T22:49:53.603Z----------------------------------------------------------------
213502024-02-20T22:49:53.603Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
213512024-02-20T22:49:53.604ZStates: Active Active Active
213522024-02-20T22:49:53.604ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213532024-02-20T22:49:53.604Z 1 Acked 1000 Write 3 Done Sent Done false
213542024-02-20T22:49:53.604Z 2 Acked 1001 Read 9 Done Sent Done false
213552024-02-20T22:49:53.604Z 3 Acked 1002 Read 1 Done Sent Done false
213562024-02-20T22:49:53.604Z 4 Acked 1003 Write 10 Done Sent Done false
213572024-02-20T22:49:53.604Z 5 Acked 1004 Read 8 Done Sent Done false
213582024-02-20T22:49:53.604Z 6 Acked 1005 Flush 0 Done Sent Done false
213592024-02-20T22:49:53.605Z 7 Acked 1006 Write 6 Done Sent Done false
213602024-02-20T22:49:53.605Z 8 Acked 1007 Read 5 Done Sent Done false
213612024-02-20T22:49:53.605Z 9 Acked 1008 Read 6 Done Sent Done false
213622024-02-20T22:49:53.605Z 10 Acked 1009 Write 3 Done Sent Done false
213632024-02-20T22:49:53.605Z 11 Acked 1010 Write 6 Done Sent Done false
213642024-02-20T22:49:53.605Z 12 Acked 1011 Write 1 Done Sent Done false
213652024-02-20T22:49:53.605Z 13 Acked 1012 Read 7 Done Sent Done false
213662024-02-20T22:49:53.605Z 14 Acked 1013 Read 5 Done Sent Done false
213672024-02-20T22:49:53.605Z 15 Acked 1014 Write 6 Done Sent Done false
213682024-02-20T22:49:53.605Z 16 Acked 1015 Write 3 Done Sent Done false
213692024-02-20T22:49:53.605Z 17 Acked 1016 Write 2 Done Sent Done false
213702024-02-20T22:49:53.605Z 18 Acked 1017 Write 10 Done Sent Done false
213712024-02-20T22:49:53.606Z 19 Acked 1018 Read 10 Done Sent Done false
213722024-02-20T22:49:53.606Z 20 Acked 1019 Write 2 Done Sent Done false
213732024-02-20T22:49:53.606Z 21 Acked 1020 Read 3 Done Sent Done false
213742024-02-20T22:49:53.606Z 22 Acked 1021 Read 7 Done Sent Done false
213752024-02-20T22:49:53.606Z 23 Acked 1022 Write 6 Done Sent Done false
213762024-02-20T22:49:53.606Z 24 Acked 1023 Write 6 Done Sent Done false
213772024-02-20T22:49:53.606Z 25 Acked 1024 Read 6 Done Sent Done false
213782024-02-20T22:49:53.606Z 26 Acked 1025 Write 6 Done Sent Done false
213792024-02-20T22:49:53.606Z 27 Acked 1026 Read 2 Done Sent Done false
213802024-02-20T22:49:53.606Z 28 Acked 1027 Write 8 Done Sent Done false
213812024-02-20T22:49:53.607Z 29 Acked 1028 Read 6 Done Sent Done false
213822024-02-20T22:49:53.607Z 30 Acked 1029 Write 2 Sent Sent Sent false
213832024-02-20T22:49:53.607Z STATES DS:0 DS:1 DS:2 TOTAL
213842024-02-20T22:49:53.607Z New 0 0 0 0
213852024-02-20T22:49:53.607Z Sent 1 30 1 32
213862024-02-20T22:49:53.607Z Done 29 0 29 58
213872024-02-20T22:49:53.607Z Skipped 0 0 0 0
213882024-02-20T22:49:53.607Z Error 0 0 0 0
213892024-02-20T22:49:53.607ZLast Flush: 1005 0 1005
213902024-02-20T22:49:53.607ZDownstairs last five completed:
213912024-02-20T22:49:53.608ZUpstairs last five completed: 30 29 28 27 26
213922024-02-20T22:49:53.608Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-20T22:49:52.704669531Z","hostname":"ip-10-150-1-191.us-west-2.compute.internal","pid":1930}
213932024-02-20T22:49:53.608ZWrote out file "/tmp/test_repair_verify.data"
213942024-02-20T22:49:53.608Z
213952024-02-20T22:49:53.608ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213962024-02-20T22:49:53.608ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213972024-02-20T22:49:53.608Z 0 000-019 192 192 192 332 332 332 T F T <---
213982024-02-20T22:49:53.608Z 1 020-039 196 196 196 343 343 343 T F T <---
213992024-02-20T22:49:53.608Z 2 040-059 191 191 191 331 331 331 T F T <---
214002024-02-20T22:49:53.608Z 3 060-079 197 197 197 346 346 346 F F F
214012024-02-20T22:49:53.608Z 4 080-099 197 197 197 346 346 346 T F T <---
214022024-02-20T22:49:53.609Z 5 100-119 196 196 196 345 345 345 F F F
214032024-02-20T22:49:53.609Z 6 120-139 195 195 195 342 342 342 F F F
214042024-02-20T22:49:53.609Z 7 140-159 194 194 194 339 339 339 F F F
214052024-02-20T22:49:53.609Z 8 160-179 196 196 196 345 345 345 T F T <---
214062024-02-20T22:49:53.609Z 9 180-199 191 191 191 331 331 331 T F T <---
214072024-02-20T22:49:53.609Z 10 200-219 195 195 195 342 342 342 T F T <---
214082024-02-20T22:49:53.609Z 11 220-239 198 194 198 347 338 347 F F F <---
214092024-02-20T22:49:53.609Z 12 240-259 196 196 196 345 345 345 F F F
214102024-02-20T22:49:53.609Z 13 260-279 196 196 196 345 345 345 F F F
214112024-02-20T22:49:53.610Z 14 280-299 197 197 197 346 346 346 F F F
214122024-02-20T22:49:53.610Z 15 300-319 197 197 197 346 346 346 T F T <---
214132024-02-20T22:49:53.610Z 16 320-339 196 196 196 345 345 345 F F F
214142024-02-20T22:49:53.610Z 17 340-359 192 192 192 332 332 332 F F F
214152024-02-20T22:49:53.610Z 18 360-379 196 196 196 345 345 345 F F F
214162024-02-20T22:49:53.610Z 19 380-399 196 196 196 345 345 345 F F F
214172024-02-20T22:49:53.610Z 20 400-419 197 197 197 346 346 346 F F F
214182024-02-20T22:49:53.610Z 21 420-439 190 190 190 328 328 328 T F T <---
214192024-02-20T22:49:53.610Z 22 440-459 196 196 196 344 344 344 T F T <---
214202024-02-20T22:49:53.620Z 23 460-479 196 196 196 344 344 344 F F F
214212024-02-20T22:49:53.620Z 24 480-499 193 193 193 337 337 337 F F F
214222024-02-20T22:49:53.620Z 25 500-519 198 190 198 347 330 347 F F F <---
214232024-02-20T22:49:53.620Z 26 520-539 196 196 196 345 345 345 F F F
214242024-02-20T22:49:53.620Z 27 540-559 196 196 196 345 345 345 F F F
214252024-02-20T22:49:53.620Z 28 560-579 195 195 195 342 342 342 F F F
214262024-02-20T22:49:53.620Z 29 580-599 197 197 197 346 346 346 F F F
214272024-02-20T22:49:53.620ZMax gen: 198, Max flush: 347
214282024-02-20T22:49:53.620ZError: Difference in extent metadata found!
214292024-02-20T22:49:53.620ZA Difference in extent metadata is expected here
214302024-02-20T22:49:53.620ZOn loop 99
214312024-02-20T22:49:53.620Z
214322024-02-20T22:49:53.620ZVerifying data now
214332024-02-20T22:49:57.860ZNo change after verify
214342024-02-20T22:49:57.874ZLoop: 99 Downstairs dump after verify (and repair):
214352024-02-20T22:49:58.547ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214362024-02-20T22:49:58.548Z 0 000-019 199 199 199 348 348 348 F F F
214372024-02-20T22:49:58.548Z 1 020-039 199 199 199 348 348 348 F F F
214382024-02-20T22:49:58.548Z 2 040-059 199 199 199 348 348 348 F F F
214392024-02-20T22:49:58.548Z 3 060-079 197 197 197 346 346 346 F F F
214402024-02-20T22:49:58.548Z 4 080-099 199 199 199 348 348 348 F F F
214412024-02-20T22:49:58.548Z 5 100-119 196 196 196 345 345 345 F F F
214422024-02-20T22:49:58.548Z 6 120-139 195 195 195 342 342 342 F F F
214432024-02-20T22:49:58.548Z 7 140-159 194 194 194 339 339 339 F F F
214442024-02-20T22:49:58.548Z 8 160-179 199 199 199 348 348 348 F F F
214452024-02-20T22:49:58.549Z 9 180-199 199 199 199 348 348 348 F F F
214462024-02-20T22:49:58.549Z 10 200-219 199 199 199 348 348 348 F F F
214472024-02-20T22:49:58.549Z 11 220-239 198 198 198 347 347 347 F F F
214482024-02-20T22:49:58.549Z 12 240-259 196 196 196 345 345 345 F F F
214492024-02-20T22:49:58.549Z 13 260-279 196 196 196 345 345 345 F F F
214502024-02-20T22:49:58.549Z 14 280-299 197 197 197 346 346 346 F F F
214512024-02-20T22:49:58.549Z 15 300-319 199 199 199 348 348 348 F F F
214522024-02-20T22:49:58.549Z 16 320-339 196 196 196 345 345 345 F F F
214532024-02-20T22:49:58.550Z 17 340-359 192 192 192 332 332 332 F F F
214542024-02-20T22:49:58.550Z 18 360-379 196 196 196 345 345 345 F F F
214552024-02-20T22:49:58.550Z 19 380-399 196 196 196 345 345 345 F F F
214562024-02-20T22:49:58.550Z 20 400-419 197 197 197 346 346 346 F F F
214572024-02-20T22:49:58.550Z 21 420-439 199 199 199 348 348 348 F F F
214582024-02-20T22:49:58.550Z 22 440-459 199 199 199 348 348 348 F F F
214592024-02-20T22:49:58.551Z 23 460-479 196 196 196 344 344 344 F F F
214602024-02-20T22:49:58.551Z 24 480-499 193 193 193 337 337 337 F F F
214612024-02-20T22:49:58.551Z 25 500-519 198 198 198 347 347 347 F F F
214622024-02-20T22:49:58.551Z 26 520-539 196 196 196 345 345 345 F F F
214632024-02-20T22:49:58.551Z 27 540-559 196 196 196 345 345 345 F F F
214642024-02-20T22:49:58.551Z 28 560-579 195 195 195 342 342 342 F F F
214652024-02-20T22:49:58.552Z 29 580-599 197 197 197 346 346 346 F F F
214662024-02-20T22:49:58.552ZMax gen: 199, Max flush: 348
214672024-02-20T22:49:58.552Z8:14 Test duration
214682024-02-20T22:49:58.552ZTest completed
214692024-02-20T22:49:58.552Z
214702024-02-20T22:49:58.552Zreal 8:13.567489031
214712024-02-20T22:49:58.552Zuser 1:38.678050338
214722024-02-20T22:49:58.552Zsys 17.750465804
214732024-02-20T22:49:58.552Ztrap 0.087225377
214742024-02-20T22:49:58.553Ztflt 0.044263501
214752024-02-20T22:49:58.553Zdflt 0.049702361
214762024-02-20T22:49:58.553Zkflt 0.002940497
214772024-02-20T22:49:58.553Zlock 9:28:07.181882829
214782024-02-20T22:49:58.553Zslp 39:18.741693261
214792024-02-20T22:49:58.553Zlat 6.662749516
214802024-02-20T22:49:58.554Zstop 0.047527813
214812024-02-20T22:49:58.554Zprocess exited: duration 502497 ms, exit code 0
 
214822024-02-20T22:49:58.581Zfound 4 output files
214832024-02-20T22:49:58.581Zuploading: /tmp/test_repair_ds8810.txt (278610 bytes)
214842024-02-20T22:50:00.749Zuploaded: /tmp/test_repair_ds8810.txt
214852024-02-20T22:50:00.749Zuploading: /tmp/test_repair_ds8820.txt (19753 bytes)
214862024-02-20T22:50:02.278Zuploaded: /tmp/test_repair_ds8820.txt
214872024-02-20T22:50:02.278Zuploading: /tmp/test_repair_ds8830.txt (44153 bytes)
214882024-02-20T22:50:03.323Zuploaded: /tmp/test_repair_ds8830.txt
214892024-02-20T22:50:03.323Zuploading: /tmp/test_repair_out.txt (82209 bytes)
214902024-02-20T22:50:04.427Zuploaded: /tmp/test_repair_out.txt