01HP2CJQTNWHSER011NYFM1351: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HP2CK6708RSR6MEX7M727P76

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-07T18:51:41.952Zjob dependencies complete; ready to run (waiting for 34 m 41 s)
22024-02-07T18:52:42.179Zjob assigned to worker 01HP2EK0R4YE05TYE19MH83H7M (queued for 1 m 0 s)
32024-02-07T18:52:42.419Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-07T18:52:54.200Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-07T18:52:54.200Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-07T18:53:09.977Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-07T18:53:09.977Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-07T18:53:31.036Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-07T18:53:31.036Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-07T18:53:44.157Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-07T18:53:44.162Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-07T18:53:44.176Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-07T18:53:44.176Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-07T18:53:44.200Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-07T18:53:44.200Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-07T18:53:44.234Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-07T18:53:44.234Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-07T18:53:44.269Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-07T18:53:44.269Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-07T18:53:44.711Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-07T18:53:44.711Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-07T18:53:44.798Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-07T18:53:44.798Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-07T18:53:44.821Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-07T18:53:44.825Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-07T18:54:11.628Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-07T18:54:11.628Zstarting task 0: "setup"
282024-02-07T18:54:11.862Z++ uname -s
292024-02-07T18:54:11.865Z+ kern=SunOS
302024-02-07T18:54:11.865Z+ case "$kern" in
312024-02-07T18:54:11.865Z+ groupadd -g 12345 build
322024-02-07T18:54:11.865Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-07T18:54:13.880Z+ zfs create -o mountpoint=/work rpool/work
342024-02-07T18:54:13.965Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-07T18:54:13.976Z+ home_fs=zfs
362024-02-07T18:54:13.976Z+ [[ zfs == autofs ]]
372024-02-07T18:54:13.976Z+ mkdir -p /home/build
382024-02-07T18:54:13.976Z+ chown build:build /home/build /work
392024-02-07T18:54:15.981Z+ chmod 0700 /home/build /work
402024-02-07T18:54:16.014Zprocess exited: duration 4363 ms, exit code 0
 
412024-02-07T18:54:16.470Zstarting task 1: "authentication"
422024-02-07T18:54:16.487Zprocess exited: duration 20 ms, exit code 0
 
432024-02-07T18:54:16.495Zstarting task 2: "build"
442024-02-07T18:54:16.500Z+ banner cores
452024-02-07T18:54:16.536Z
462024-02-07T18:54:16.540Z #### #### ##### ###### ####
472024-02-07T18:54:16.540Z # # # # # # # #
482024-02-07T18:54:16.540Z # # # # # ##### ####
492024-02-07T18:54:16.540Z # # # ##### # #
502024-02-07T18:54:16.540Z # # # # # # # # #
512024-02-07T18:54:16.540Z #### #### # # ###### ####
522024-02-07T18:54:16.540Z
532024-02-07T18:54:16.540Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-07T18:54:16.546Z+ echo 'input bins dir contains:'
552024-02-07T18:54:16.547Zinput bins dir contains:
562024-02-07T18:54:16.550Z+ ls -ltr /input/build/work/bins
572024-02-07T18:54:16.550Ztotal 759567
582024-02-07T18:54:16.550Z-rw-r--r-- 1 root root 120910814 Feb 7 18:52 crucible-downstairs.gz
592024-02-07T18:54:16.550Z-rw-r--r-- 1 root root 91082247 Feb 7 18:53 crucible-hammer.gz
602024-02-07T18:54:16.550Z-rw-r--r-- 1 root root 104703415 Feb 7 18:53 crutest.gz
612024-02-07T18:54:16.550Z-rw-r--r-- 1 root root 71941606 Feb 7 18:53 dsc.gz
622024-02-07T18:54:16.550Z+ echo 'input script dir contains:'
632024-02-07T18:54:16.550Zinput script dir contains:
642024-02-07T18:54:16.550Z+ ls -ltr /input/build/work/scripts
652024-02-07T18:54:16.554Ztotal 38
662024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 3056 Feb 7 18:53 perf-downstairs-tick.d
672024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 939 Feb 7 18:53 test_ds.sh
682024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 3746 Feb 7 18:53 test_live_repair.sh
692024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 6416 Feb 7 18:53 test_repair.sh
702024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 2969 Feb 7 18:53 test_replay.sh
712024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 9465 Feb 7 18:53 test_up.sh
722024-02-07T18:54:16.554Z-rw-r--r-- 1 root root 2824 Feb 7 18:53 upstairs_info.d
732024-02-07T18:54:16.554Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-07T18:54:16.561Z+ echo ' chmod input script dir contains:'
752024-02-07T18:54:16.561Z+ ls -ltr /input/build/work/scripts
762024-02-07T18:54:16.561Z chmod input script dir contains:
772024-02-07T18:54:16.561Ztotal 38
782024-02-07T18:54:16.561Z-rwxr-xr-x 1 root root 3056 Feb 7 18:53 perf-downstairs-tick.d
792024-02-07T18:54:16.562Z-rwxr-xr-x 1 root root 939 Feb 7 18:53 test_ds.sh
802024-02-07T18:54:16.562Z-rwxr-xr-x 1 root root 3746 Feb 7 18:53 test_live_repair.sh
812024-02-07T18:54:16.562Z-rwxr-xr-x 1 root root 6416 Feb 7 18:53 test_repair.sh
822024-02-07T18:54:16.562Z-rwxr-xr-x 1 root root 2969 Feb 7 18:53 test_replay.sh
832024-02-07T18:54:16.562Z-rwxr-xr-x 1 root root 9465 Feb 7 18:53 test_up.sh
842024-02-07T18:54:16.562Z-rwxr-xr-x 1 root root 2824 Feb 7 18:53 upstairs_info.d
852024-02-07T18:54:16.562Z+ banner unpack
862024-02-07T18:54:16.565Z
872024-02-07T18:54:16.565Z # # # # ##### ## #### # #
882024-02-07T18:54:16.566Z # # ## # # # # # # # # #
892024-02-07T18:54:16.566Z # # # # # # # # # # ####
902024-02-07T18:54:16.566Z # # # # # ##### ###### # # #
912024-02-07T18:54:16.566Z # # # ## # # # # # # #
922024-02-07T18:54:16.566Z #### # # # # # #### # #
932024-02-07T18:54:16.566Z
942024-02-07T18:54:16.566Z+ mkdir -p /var/tmp/bins
952024-02-07T18:54:16.566Z+ for t in "$input/bins/"*.gz
962024-02-07T18:54:16.566Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-07T18:54:16.566Z+ b=crucible-downstairs.gz
982024-02-07T18:54:16.566Z+ b=crucible-downstairs
992024-02-07T18:54:16.566Z+ gunzip
1002024-02-07T18:54:19.423Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-07T18:54:19.430Z+ for t in "$input/bins/"*.gz
1022024-02-07T18:54:19.430Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-07T18:54:19.430Z+ b=crucible-hammer.gz
1042024-02-07T18:54:19.431Z+ b=crucible-hammer
1052024-02-07T18:54:19.431Z+ gunzip
1062024-02-07T18:54:21.519Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-07T18:54:21.531Z+ for t in "$input/bins/"*.gz
1082024-02-07T18:54:21.531Z++ basename /input/build/work/bins/crutest.gz
1092024-02-07T18:54:21.531Z+ b=crutest.gz
1102024-02-07T18:54:21.531Z+ b=crutest
1112024-02-07T18:54:21.531Z+ gunzip
1122024-02-07T18:54:23.954Z+ chmod +x /var/tmp/bins/crutest
1132024-02-07T18:54:23.963Z+ for t in "$input/bins/"*.gz
1142024-02-07T18:54:23.963Z++ basename /input/build/work/bins/dsc.gz
1152024-02-07T18:54:23.964Z+ b=dsc.gz
1162024-02-07T18:54:23.964Z+ b=dsc
1172024-02-07T18:54:23.964Z+ gunzip
1182024-02-07T18:54:25.573Z+ chmod +x /var/tmp/bins/dsc
1192024-02-07T18:54:25.576Z+ export BINDIR=/var/tmp/bins
1202024-02-07T18:54:25.576Z+ BINDIR=/var/tmp/bins
1212024-02-07T18:54:25.576Z+ banner repair
1222024-02-07T18:54:25.579Z
1232024-02-07T18:54:25.579Z ##### ###### ##### ## # #####
1242024-02-07T18:54:25.579Z # # # # # # # # # #
1252024-02-07T18:54:25.580Z # # ##### # # # # # # #
1262024-02-07T18:54:25.580Z ##### # ##### ###### # #####
1272024-02-07T18:54:25.580Z # # # # # # # # #
1282024-02-07T18:54:25.580Z # # ###### # # # # # #
1292024-02-07T18:54:25.580Z
1302024-02-07T18:54:25.580Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-07T18:54:25.787ZTurn off color for downstairs dump
1322024-02-07T18:54:25.800ZCreating region directory at: /var/tmp/test_repair
1332024-02-07T18:54:25.803ZCreating dsc directory at: /tmp/dsc
1342024-02-07T18:54:27.692ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 1.8852925
1352024-02-07T18:54:27.760ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.06976217
1362024-02-07T18:54:27.798ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.037524465
1372024-02-07T18:54:27.798ZRegion set with 3 regions was created
1382024-02-07T18:54:27.815Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-07T18:54:27.815ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-07T18:54:27.840Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.113061217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141}
1412024-02-07T18:54:27.841Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.113724607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141}
1422024-02-07T18:54:27.841Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.113744374Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141}
1432024-02-07T18:54:28.276Z{"msg":"Crucible 3e09a837-627b-419d-8b25-12baf18831ba has session id: d8376230-a9d4-403f-964b-f92c0130af02","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.113759972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1442024-02-07T18:54:28.276Z{"msg":"Upstairs opts: Upstairs UUID: 3e09a837-627b-419d-8b25-12baf18831ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.113794807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1452024-02-07T18:54:28.276Z{"msg":"Crucible stats registered with UUID: 3e09a837-627b-419d-8b25-12baf18831ba","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.113820134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1462024-02-07T18:54:28.277ZCrucible runtime is spawned
1472024-02-07T18:54:28.277Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114317117Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141}
1482024-02-07T18:54:28.277Z{"msg":"3e09a837-627b-419d-8b25-12baf18831ba active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114548495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1492024-02-07T18:54:28.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114580951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1502024-02-07T18:54:28.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114598099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1512024-02-07T18:54:28.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114613496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1522024-02-07T18:54:28.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114640123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1532024-02-07T18:54:28.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114675318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1542024-02-07T18:54:28.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114692696Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1552024-02-07T18:54:28.278Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114763146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1562024-02-07T18:54:28.278Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114801391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1572024-02-07T18:54:28.278Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114847875Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1582024-02-07T18:54:28.278Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.114974658Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1592024-02-07T18:54:28.278Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.115036079Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1602024-02-07T18:54:28.278Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.115127707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1612024-02-07T18:54:28.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.11761963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1622024-02-07T18:54:28.279Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.117655635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1632024-02-07T18:54:28.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.117735604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1642024-02-07T18:54:28.279Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.117820893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1652024-02-07T18:54:28.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.117861307Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1662024-02-07T18:54:28.279Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.117892123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1672024-02-07T18:54:28.280Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.118967318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1682024-02-07T18:54:28.280Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.119014921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1692024-02-07T18:54:28.280Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.119091691Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1702024-02-07T18:54:28.280Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120108613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1712024-02-07T18:54:28.280Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120152087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1722024-02-07T18:54:28.281Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120168565Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1732024-02-07T18:54:28.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120182343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1742024-02-07T18:54:28.281Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.1202044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1752024-02-07T18:54:28.281Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120219118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1762024-02-07T18:54:28.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120231897Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1772024-02-07T18:54:28.282Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120264262Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1782024-02-07T18:54:28.282Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120432889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1792024-02-07T18:54:28.282Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120460256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1802024-02-07T18:54:28.282Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120584849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1812024-02-07T18:54:28.282Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120610845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1822024-02-07T18:54:28.282Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120625434Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1832024-02-07T18:54:28.282Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120646561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1842024-02-07T18:54:28.283Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120660829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1852024-02-07T18:54:28.283Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120680866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1862024-02-07T18:54:28.283Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120695394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1872024-02-07T18:54:28.283Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120715261Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1882024-02-07T18:54:28.283Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120729399Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1892024-02-07T18:54:28.284Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120749827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1902024-02-07T18:54:28.284Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120764635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1912024-02-07T18:54:28.284ZWait for a query_work_queue command to finish before sending IO
1922024-02-07T18:54:28.284Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120792191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1932024-02-07T18:54:28.284Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120812678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1942024-02-07T18:54:28.284Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120836735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1952024-02-07T18:54:28.284Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120850623Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1962024-02-07T18:54:28.285Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120863971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1972024-02-07T18:54:28.285Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120884088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1982024-02-07T18:54:28.285Z{"msg":"3e09a837-627b-419d-8b25-12baf18831ba is now active with session: d8376230-a9d4-403f-964b-f92c0130af02","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120897247Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
1992024-02-07T18:54:28.285Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120909145Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2002024-02-07T18:54:28.285Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:27.120928633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141}
2012024-02-07T18:54:28.285ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-07T18:54:28.286ZFill test
2032024-02-07T18:54:28.712ZRead and Verify all blocks (0..600 range:false)
2042024-02-07T18:54:28.713ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-07T18:54:28.713ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-07T18:54:28.713Z----------------------------------------------------------------
2072024-02-07T18:54:28.713Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-07T18:54:28.713ZStates: Active Active Active
2092024-02-07T18:54:28.713ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-07T18:54:28.713Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-07T18:54:28.713Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-07T18:54:28.714Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-07T18:54:28.714Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-07T18:54:28.714Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-07T18:54:28.714Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-07T18:54:28.714Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-07T18:54:28.714Z New 0 0 0 0
2182024-02-07T18:54:28.714Z Sent 0 0 0 0
2192024-02-07T18:54:28.714Z Done 6 6 6 18
2202024-02-07T18:54:28.715Z Skipped 0 0 0 0
2212024-02-07T18:54:28.715Z Error 0 0 0 0
2222024-02-07T18:54:28.715ZLast Flush: 1006 1006 1006
2232024-02-07T18:54:28.715ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-07T18:54:28.715ZUpstairs last five completed: 13 12 11 10 9
2252024-02-07T18:54:28.715ZCLIENT: Up:0 ds:6 act:3
2262024-02-07T18:54:32.081Z----------------------------------------------------------------
2272024-02-07T18:54:32.082Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-07T18:54:32.082ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-07T18:54:32.082ZUpstairs last five completed: 14 13 12 11 10
2302024-02-07T18:54:32.089ZCLIENT: Up:0 ds:0 act:3
2312024-02-07T18:54:32.089ZCLIENT: All crucible jobs finished, exiting program
2322024-02-07T18:54:32.089Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358037011Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141}
2332024-02-07T18:54:32.089Z{"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358170712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2342024-02-07T18:54:32.090Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358212087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2352024-02-07T18:54:32.090Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358247442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2362024-02-07T18:54:32.090Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358276738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2372024-02-07T18:54:32.090Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358324712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2382024-02-07T18:54:32.091Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.358357637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2392024-02-07T18:54:32.091Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358390993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2402024-02-07T18:54:32.091Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358421539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2412024-02-07T18:54:32.091Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358451055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2422024-02-07T18:54:32.092Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.358480641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2432024-02-07T18:54:32.092Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.358508877Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2442024-02-07T18:54:32.092Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.358540133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"d8376230-a9d4-403f-964b-f92c0130af02"}
2452024-02-07T18:54:32.098Z
2462024-02-07T18:54:32.098ZBegin loop 1
2472024-02-07T18:54:32.099ZDownstairs to restart: 2
2482024-02-07T18:54:32.107Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.380950631Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143}
2492024-02-07T18:54:32.107Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381198518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143}
2502024-02-07T18:54:32.107Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381224534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143}
2512024-02-07T18:54:32.126Z{"msg":"Crucible 523b32e1-5f47-4cd9-918a-de3770c10d05 has session id: 4b4b87dc-c55b-46c2-86e9-ab12fd0bf978","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38125803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2522024-02-07T18:54:32.127Z{"msg":"Upstairs opts: Upstairs UUID: 523b32e1-5f47-4cd9-918a-de3770c10d05, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381303164Z","hostname":"Crucible runtime is spawned
2532024-02-07T18:54:32.127Zip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2542024-02-07T18:54:32.127Z{"msg":"Crucible stats registered with UUID: 523b32e1-5f47-4cd9-918a-de3770c10d05","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381350888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2552024-02-07T18:54:32.127Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381384683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143}
2562024-02-07T18:54:32.128Z{"msg":"523b32e1-5f47-4cd9-918a-de3770c10d05 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38147748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2572024-02-07T18:54:32.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381543411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2582024-02-07T18:54:32.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381588885Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2592024-02-07T18:54:32.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381634999Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2602024-02-07T18:54:32.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381656996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2612024-02-07T18:54:32.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381687202Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2622024-02-07T18:54:32.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381713228Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2632024-02-07T18:54:32.129Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381738345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2642024-02-07T18:54:32.129Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381764441Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2652024-02-07T18:54:32.130Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381790488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2662024-02-07T18:54:32.130Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381899783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2672024-02-07T18:54:32.130Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.381958525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2682024-02-07T18:54:32.130Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38199766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2692024-02-07T18:54:32.130Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.382784114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2702024-02-07T18:54:32.131Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.382822728Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2712024-02-07T18:54:32.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.382851405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2722024-02-07T18:54:32.131Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38288884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2732024-02-07T18:54:32.131Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.383481889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2742024-02-07T18:54:32.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.383651356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2752024-02-07T18:54:32.132Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38369721Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2762024-02-07T18:54:32.132Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.383726796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2772024-02-07T18:54:32.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384084118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2782024-02-07T18:54:32.132Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384120523Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2792024-02-07T18:54:32.133Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384136921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2802024-02-07T18:54:32.133Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384273832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2812024-02-07T18:54:32.133Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384306138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2822024-02-07T18:54:32.133Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384321306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2832024-02-07T18:54:32.133Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384335424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2842024-02-07T18:54:32.133Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38480989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2852024-02-07T18:54:32.133Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384836076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2862024-02-07T18:54:32.133Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384851664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2872024-02-07T18:54:32.134Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384865422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2882024-02-07T18:54:32.134Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38488363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2892024-02-07T18:54:32.134Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384905587Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2902024-02-07T18:54:32.134Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384919705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2912024-02-07T18:54:32.134Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.384940362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2922024-02-07T18:54:32.134Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38495462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2932024-02-07T18:54:32.134Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38502636Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2942024-02-07T18:54:32.135Z{"msg":"Wait for a query_work_queue command to finish before sending IO
2952024-02-07T18:54:32.135Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385047448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2962024-02-07T18:54:32.135Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385069234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2972024-02-07T18:54:32.135Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385083593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2982024-02-07T18:54:32.135Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38510312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
2992024-02-07T18:54:32.136Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385116838Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3002024-02-07T18:54:32.136Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385131376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3012024-02-07T18:54:32.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385150534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3022024-02-07T18:54:32.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385164182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3032024-02-07T18:54:32.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385184359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3042024-02-07T18:54:32.136Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385198097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3052024-02-07T18:54:32.137Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385217234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3062024-02-07T18:54:32.137Z{"msg":"523b32e1-5f47-4cd9-918a-de3770c10d05 is now active with session: 4b4b87dc-c55b-46c2-86e9-ab12fd0bf978","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385230653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3072024-02-07T18:54:32.137Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.38524914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143,"session_id":"4b4b87dc-c55b-46c2-86e9-ab12fd0bf978"}
3082024-02-07T18:54:32.137Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:31.385261719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143}
3092024-02-07T18:54:32.137ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3102024-02-07T18:54:32.137ZLoading write count information from file "/tmp/test_repair_verify.data"
3112024-02-07T18:54:32.138ZRun Repair workload
3122024-02-07T18:54:32.138Z01/30 Write block 203 len 24576 data: 2 2 2 2 2 2
3132024-02-07T18:54:32.138Z02/30 Read block 150 len 20480
3142024-02-07T18:54:32.138Z03/30 Write block 345 len 8192 data: 2 2
3152024-02-07T18:54:32.138Z04/30 Read block 201 len 16384
3162024-02-07T18:54:32.138Z05/30 Flush
3172024-02-07T18:54:32.138Z06/30 Flush
3182024-02-07T18:54:32.139Z07/30 Write block 320 len 4096 data: 2
3192024-02-07T18:54:32.139Z08/30 Read block 171 len 8192
3202024-02-07T18:54:32.139Z09/30 Flush
3212024-02-07T18:54:32.148Z10/30 Write block 541 len 8192 data: 2 2
3222024-02-07T18:54:32.148Z11/30 Write block 191 len 36864 data: 2 2 2 2 2 2 2 2 2
3232024-02-07T18:54:32.149Z12/30 Read block 69 len 8192
3242024-02-07T18:54:32.149Z13/30 Flush
3252024-02-07T18:54:32.149Z14/30 Write block 128 len 16384 data: 2 2 2 2
3262024-02-07T18:54:32.149Z15/30 Write block 436 len 8192 data: 2 2
3272024-02-07T18:54:32.149Z16/30 Write block 158 len 24576 data: 2 2 2 2 2 2
3282024-02-07T18:54:32.149Z17/30 Write block 50 len 20480 data: 2 2 2 2 2
3292024-02-07T18:54:32.149Z18/30 Write block 164 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3302024-02-07T18:54:32.150Z19/30 Read block 78 len 24576
3312024-02-07T18:54:32.150Z20/30 Write block 189 len 16384 data: 2 2 3 3
3322024-02-07T18:54:32.150Z21/30 Read block 577 len 20480
3332024-02-07T18:54:32.150Z22/30 Write block 158 len 28672 data: 3 3 3 3 3 3 3
3342024-02-07T18:54:32.150Z23/30 Write block 78 len 8192 data: 2 2
3352024-02-07T18:54:32.150Z24/30 Read block 9 len 32768
3362024-02-07T18:54:32.150Z25/30 Write block 170 len 32768 data: 3 3 3 3 2 2 2 2
3372024-02-07T18:54:32.150Z26/30 Read block 129 len 12288
3382024-02-07T18:54:32.151Z27/30 Write block 51 len 36864 data: 3 3 3 3 2 2 2 2 2
3392024-02-07T18:54:32.151Z28/30 Write block 165 len 28672 data: 3 3 3 3 3 4 4
3402024-02-07T18:54:32.151Z29/30 Read block 403 len 40960
3412024-02-07T18:54:32.151Z30/30 Read block 140 len 4096
3422024-02-07T18:54:32.151Z----------------------------------------------------------------
3432024-02-07T18:54:32.152Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3442024-02-07T18:54:32.152ZStates: Active Active Active
3452024-02-07T18:54:32.152ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3462024-02-07T18:54:32.152Z 1 Acked 1000 Write 6 Done Done Sent false
3472024-02-07T18:54:32.152Z 2 Acked 1001 Read 5 Done Done Sent false
3482024-02-07T18:54:32.152Z 3 Acked 1002 Write 2 Done Done Sent false
3492024-02-07T18:54:32.153Z 4 Acked 1003 Read 4 Done Done Sent false
3502024-02-07T18:54:32.153Z 5 Acked 1004 Flush 0 Done Done Sent false
3512024-02-07T18:54:32.153Z 6 Acked 1005 Flush 0 Done Done Sent false
3522024-02-07T18:54:32.153Z 7 Acked 1006 Write 1 Done Done Sent false
3532024-02-07T18:54:32.153Z 8 Acked 1007 Read 2 Done Done Sent false
3542024-02-07T18:54:32.153Z 9 Acked 1008 Flush 0 Done Done Sent false
3552024-02-07T18:54:32.153Z 10 Acked 1009 Write 2 Done Done Sent false
3562024-02-07T18:54:32.154Z 11 Acked 1010 Write 9 Done Done Sent false
3572024-02-07T18:54:32.154Z 12 Acked 1011 Read 2 Done Done Sent false
3582024-02-07T18:54:32.154Z 13 Acked 1012 Flush 0 Done Done Sent false
3592024-02-07T18:54:32.154Z 14 Acked 1013 Write 4 Done Done Sent false
3602024-02-07T18:54:32.154Z 15 Acked 1014 Write 2 Done Done Sent false
3612024-02-07T18:54:32.154Z 16 Acked 1015 Write 6 Done Done Sent false
3622024-02-07T18:54:32.154Z 17 Acked 1016 Write 5 Done Done Sent false
3632024-02-07T18:54:32.154Z 18 Acked 1017 Write 10 Done Done Sent false
3642024-02-07T18:54:32.155Z 19 Acked 1018 Read 6 Done Done Sent false
3652024-02-07T18:54:32.155Z 20 Acked 1019 Write 4 Done Done Sent false
3662024-02-07T18:54:32.155Z 21 Acked 1020 Read 5 Done Done Sent false
3672024-02-07T18:54:32.155Z 22 Acked 1021 Write 7 Done Done Sent false
3682024-02-07T18:54:32.155Z 23 Acked 1022 Write 2 Done Done Sent false
3692024-02-07T18:54:32.155Z 24 Acked 1023 Read 8 Done Done Sent false
3702024-02-07T18:54:32.155Z 25 Acked 1024 Write 8 Done Done Sent false
3712024-02-07T18:54:32.156Z 26 Acked 1025 Read 3 Done Done Sent false
3722024-02-07T18:54:32.156Z 27 Acked 1026 Write 9 Done Done Sent false
3732024-02-07T18:54:32.156Z 28 Acked 1027 Write 7 Done Done Sent false
3742024-02-07T18:54:32.156Z 29 Acked 1028 Read 10 Done Done Sent false
3752024-02-07T18:54:32.156Z 30 Acked 1029 Read 1 Done Done Sent false
3762024-02-07T18:54:32.156Z STATES DS:0 DS:1 DS:2 TOTAL
3772024-02-07T18:54:32.157Z New 0 0 0 0
3782024-02-07T18:54:32.157Z Sent 0 0 30 30
3792024-02-07T18:54:32.157Z Done 30 30 0 60
3802024-02-07T18:54:32.157Z Skipped 0 0 0 0
3812024-02-07T18:54:32.157Z Error 0 0 0 0
3822024-02-07T18:54:32.157ZLast Flush: 1012 1012 0
3832024-02-07T18:54:32.157ZDownstairs last five completed:
3842024-02-07T18:54:32.157ZUpstairs last five completed: 30 29 28 27 26
3852024-02-07T18:54:32.157Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:31.418711464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1143}
3862024-02-07T18:54:32.157ZWrote out file "/tmp/test_repair_verify.data"
3872024-02-07T18:54:32.204Z
3882024-02-07T18:54:32.204ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3892024-02-07T18:54:32.204ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3902024-02-07T18:54:32.205Z 0 000-019 1 1 1 1 1 1 F F F
3912024-02-07T18:54:32.205Z 1 020-039 1 1 1 1 1 1 F F F
3922024-02-07T18:54:32.205Z 2 040-059 1 1 1 1 1 1 T T F <---
3932024-02-07T18:54:32.205Z 3 060-079 1 1 1 1 1 1 T T F <---
3942024-02-07T18:54:32.205Z 4 080-099 1 1 1 1 1 1 F F F
3952024-02-07T18:54:32.206Z 5 100-119 1 1 1 1 1 1 F F F
3962024-02-07T18:54:32.206Z 6 120-139 1 1 1 1 1 1 T T F <---
3972024-02-07T18:54:32.206Z 7 140-159 1 1 1 1 1 1 T T F <---
3982024-02-07T18:54:32.206Z 8 160-179 1 1 1 1 1 1 T T F <---
3992024-02-07T18:54:32.206Z 9 180-199 2 2 1 5 5 1 T T F <---
4002024-02-07T18:54:32.207Z 10 200-219 2 2 1 2 2 1 F F F <---
4012024-02-07T18:54:32.207Z 11 220-239 1 1 1 1 1 1 F F F
4022024-02-07T18:54:32.207Z 12 240-259 1 1 1 1 1 1 F F F
4032024-02-07T18:54:32.207Z 13 260-279 1 1 1 1 1 1 F F F
4042024-02-07T18:54:32.207Z 14 280-299 1 1 1 1 1 1 F F F
4052024-02-07T18:54:32.208Z 15 300-319 1 1 1 1 1 1 F F F
4062024-02-07T18:54:32.208Z 16 320-339 2 2 1 4 4 1 F F F <---
4072024-02-07T18:54:32.208Z 17 340-359 2 2 1 2 2 1 F F F <---
4082024-02-07T18:54:32.208Z 18 360-379 1 1 1 1 1 1 F F F
4092024-02-07T18:54:32.208Z 19 380-399 1 1 1 1 1 1 F F F
4102024-02-07T18:54:32.208Z 20 400-419 1 1 1 1 1 1 F F F
4112024-02-07T18:54:32.208Z 21 420-439 1 1 1 1 1 1 T T F <---
4122024-02-07T18:54:32.208Z 22 440-459 1 1 1 1 1 1 F F F
4132024-02-07T18:54:32.208Z 23 460-479 1 1 1 1 1 1 F F F
4142024-02-07T18:54:32.208Z 24 480-499 1 1 1 1 1 1 F F F
4152024-02-07T18:54:32.209Z 25 500-519 1 1 1 1 1 1 F F F
4162024-02-07T18:54:32.209Z 26 520-539 1 1 1 1 1 1 F F F
4172024-02-07T18:54:32.209Z 27 540-559 2 2 1 5 5 1 F F F <---
4182024-02-07T18:54:32.209Z 28 560-579 1 1 1 1 1 1 F F F
4192024-02-07T18:54:32.209Z 29 580-599 1 1 1 1 1 1 F F F
4202024-02-07T18:54:32.209ZMax gen: 2, Max flush: 5
4212024-02-07T18:54:32.209ZError: Difference in extent metadata found!
4222024-02-07T18:54:32.209ZA Difference in extent metadata is expected here
4232024-02-07T18:54:32.209ZOn loop 1
4242024-02-07T18:54:32.209Z
4252024-02-07T18:54:32.209ZVerifying data now
4262024-02-07T18:54:36.807ZNo change after verify
4272024-02-07T18:54:36.807ZLoop: 1 Downstairs dump after verify (and repair):
4282024-02-07T18:54:36.821ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4292024-02-07T18:54:36.821Z 0 000-019 1 1 1 1 1 1 F F F
4302024-02-07T18:54:36.822Z 1 020-039 1 1 1 1 1 1 F F F
4312024-02-07T18:54:36.822Z 2 040-059 3 3 3 6 6 6 F F F
4322024-02-07T18:54:36.822Z 3 060-079 3 3 3 6 6 6 F F F
4332024-02-07T18:54:36.822Z 4 080-099 1 1 1 1 1 1 F F F
4342024-02-07T18:54:36.822Z 5 100-119 1 1 1 1 1 1 F F F
4352024-02-07T18:54:36.822Z 6 120-139 3 3 3 6 6 6 F F F
4362024-02-07T18:54:36.822Z 7 140-159 3 3 3 6 6 6 F F F
4372024-02-07T18:54:36.822Z 8 160-179 3 3 3 6 6 6 F F F
4382024-02-07T18:54:36.825Z 9 180-199 3 3 3 6 6 6 F F F
4392024-02-07T18:54:36.825Z 10 200-219 2 2 2 2 2 2 F F F
4402024-02-07T18:54:36.825Z 11 220-239 1 1 1 1 1 1 F F F
4412024-02-07T18:54:36.825Z 12 240-259 1 1 1 1 1 1 F F F
4422024-02-07T18:54:36.825Z 13 260-279 1 1 1 1 1 1 F F F
4432024-02-07T18:54:36.825Z 14 280-299 1 1 1 1 1 1 F F F
4442024-02-07T18:54:36.826Z 15 300-319 1 1 1 1 1 1 F F F
4452024-02-07T18:54:36.826Z 16 320-339 2 2 2 4 4 4 F F F
4462024-02-07T18:54:36.826Z 17 340-359 2 2 2 2 2 2 F F F
4472024-02-07T18:54:36.826Z 18 360-379 1 1 1 1 1 1 F F F
4482024-02-07T18:54:36.826Z 19 380-399 1 1 1 1 1 1 F F F
4492024-02-07T18:54:36.826Z 20 400-419 1 1 1 1 1 1 F F F
4502024-02-07T18:54:36.826Z 21 420-439 3 3 3 6 6 6 F F F
4512024-02-07T18:54:36.826Z 22 440-459 1 1 1 1 1 1 F F F
4522024-02-07T18:54:36.826Z 23 460-479 1 1 1 1 1 1 F F F
4532024-02-07T18:54:36.826Z 24 480-499 1 1 1 1 1 1 F F F
4542024-02-07T18:54:36.826Z 25 500-519 1 1 1 1 1 1 F F F
4552024-02-07T18:54:36.826Z 26 520-539 1 1 1 1 1 1 F F F
4562024-02-07T18:54:36.826Z 27 540-559 2 2 2 5 5 5 F F F
4572024-02-07T18:54:36.826Z 28 560-579 1 1 1 1 1 1 F F F
4582024-02-07T18:54:36.826Z 29 580-599 1 1 1 1 1 1 F F F
4592024-02-07T18:54:36.826ZMax gen: 3, Max flush: 6
4602024-02-07T18:54:36.826Z
4612024-02-07T18:54:36.826ZBegin loop 2
4622024-02-07T18:54:36.826ZDownstairs to restart: 1
4632024-02-07T18:54:36.843Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119019956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151}
4642024-02-07T18:54:36.843Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119301198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151}
4652024-02-07T18:54:36.848Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
4662024-02-07T18:54:36.848Z,"time":"2024-02-07T18:54:36.119348241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151}
4672024-02-07T18:54:36.848Z{"msg":"Crucible a8d089b3-5395-4f13-860f-2fda11d2069a has session id: 3036b323-dcc2-42b2-be92-7c0ad2214bf4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119390195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4682024-02-07T18:54:36.848Z{"msg":"Upstairs opts: Upstairs UUID: a8d089b3-5395-4f13-860f-2fda11d2069a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119451827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4692024-02-07T18:54:36.848Z{"msg":"Crucible stats registered with UUID: a8d089b3-5395-4f13-860f-2fda11d2069a","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119485882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4702024-02-07T18:54:36.848Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119523328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151}
4712024-02-07T18:54:36.848Z{"msg":"a8d089b3-5395-4f13-860f-2fda11d2069a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119540115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4722024-02-07T18:54:36.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119565102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4732024-02-07T18:54:36.848Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119626754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4742024-02-07T18:54:36.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.11965509Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4752024-02-07T18:54:36.848Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119681736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4762024-02-07T18:54:36.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.11972352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4772024-02-07T18:54:36.848Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119760045Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4782024-02-07T18:54:36.848Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.11980264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4792024-02-07T18:54:36.848Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119840345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4802024-02-07T18:54:36.848Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.11987616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4812024-02-07T18:54:36.849Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.119913105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4822024-02-07T18:54:36.849Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:36.119952839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4832024-02-07T18:54:36.849Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120000943Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4842024-02-07T18:54:36.849Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:36.120029009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4852024-02-07T18:54:36.849Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:36.120047586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4862024-02-07T18:54:36.849Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120076383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4872024-02-07T18:54:36.849Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120102599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4882024-02-07T18:54:36.849Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120129395Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4892024-02-07T18:54:36.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.12061003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4902024-02-07T18:54:36.849Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120665723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4912024-02-07T18:54:36.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120732344Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4922024-02-07T18:54:36.849Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.120765809Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4932024-02-07T18:54:36.849Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.121194861Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4942024-02-07T18:54:36.849Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.121364558Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4952024-02-07T18:54:36.849Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.121754346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4962024-02-07T18:54:36.849Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.121791871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4972024-02-07T18:54:36.849Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.121817427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4982024-02-07T18:54:36.849Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.121982545Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
4992024-02-07T18:54:36.849Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.122051366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5002024-02-07T18:54:36.849Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:36.122086431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5012024-02-07T18:54:37.845Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.120518452Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5022024-02-07T18:54:37.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.120717995Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5032024-02-07T18:54:37.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.12149083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5042024-02-07T18:54:37.850Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.121527295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5052024-02-07T18:54:37.850Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.12208382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5062024-02-07T18:54:37.850Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122670131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5072024-02-07T18:54:37.850Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122734422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5082024-02-07T18:54:37.850Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 6, 6, 1, 1, 6, 6, 6, 6, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122755819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5092024-02-07T18:54:37.850Z{"msg":"[0]R generation[0..12]: [1, 1, 3, 3, 1, 1, 3, 3, 3, 3, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122782506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5102024-02-07T18:54:37.850Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122801293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5112024-02-07T18:54:37.850Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 6, 6, 1, 1, 6, 6, 6, 6, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122827859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5122024-02-07T18:54:37.850Z{"msg":"[1]R generation[0..12]: [1, 1, 3, 3, 1, 1, 3, 3, 3, 3, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122845507Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5132024-02-07T18:54:37.851Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122871524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5142024-02-07T18:54:37.851Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 6, 6, 1, 1, 6, 6, 6, 6, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122887571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5152024-02-07T18:54:37.851Z{"msg":"[2]R generation[0..12]: [1, 1, 3, 3, 1, 1, 3, 3, 3, 3, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122907609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5162024-02-07T18:54:37.851Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122921817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5172024-02-07T18:54:37.851Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5182024-02-07T18:54:37.851Z,"time":"2024-02-07T18:54:37.122941794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5192024-02-07T18:54:37.851Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122958352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5202024-02-07T18:54:37.851Z{"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122978339Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5212024-02-07T18:54:37.851Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.122993987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5222024-02-07T18:54:37.851Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123014624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5232024-02-07T18:54:37.851Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123027303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5242024-02-07T18:54:37.851Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123053969Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5252024-02-07T18:54:37.851Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123068317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5262024-02-07T18:54:37.851Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123087024Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5272024-02-07T18:54:37.851Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123109761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5282024-02-07T18:54:37.851Z{"msg":"a8d089b3-5395-4f13-860f-2fda11d2069a is now active with session: 3036b323-dcc2-42b2-be92-7c0ad2214bf4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123123479Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5292024-02-07T18:54:37.851Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123135568Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151,"session_id":"3036b323-dcc2-42b2-be92-7c0ad2214bf4"}
5302024-02-07T18:54:37.851Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:37.123154705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151}
5312024-02-07T18:54:37.851ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5322024-02-07T18:54:37.851ZLoading write count information from file "/tmp/test_repair_verify.data"
5332024-02-07T18:54:37.851ZRun Repair workload
5342024-02-07T18:54:37.852Z01/30 Flush
5352024-02-07T18:54:37.855Z02/30 Write block 313 len 8192 data: 2 2
5362024-02-07T18:54:37.855Z03/30 Read block 146 len 16384
5372024-02-07T18:54:37.855Z04/30 Write block 196 len 20480 data: 3 3 3 3 2
5382024-02-07T18:54:37.855Z05/30 Write block 580 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5392024-02-07T18:54:37.855Z06/30 Write block 76 len 32768 data: 2 2 3 3 2 2 2 2
5402024-02-07T18:54:37.855Z07/30 Flush
5412024-02-07T18:54:37.858Z08/30 Write block 591 len 20480 data: 2 2 2 2 2
5422024-02-07T18:54:37.858Z09/30 Read block 210 len 40960
5432024-02-07T18:54:37.861Z10/30 Write block 118 len 16384 data: 2 2 2 2
5442024-02-07T18:54:37.861Z11/30 Flush
5452024-02-07T18:54:37.864Z12/30 Write block 64 len 28672 data: 2 2 2 2 2 2 2
5462024-02-07T18:54:37.864Z13/30 Write block 488 len 28672 data: 2 2 2 2 2 2 2
5472024-02-07T18:54:37.864Z14/30 Write block 503 len 12288 data: 2 2 2
5482024-02-07T18:54:37.864Z15/30 Read block 270 len 32768
5492024-02-07T18:54:37.867Z16/30 Read block 247 len 32768
5502024-02-07T18:54:37.867Z17/30 Flush
5512024-02-07T18:54:37.870Z18/30 Write block 376 len 32768 data: 2 2 2 2 2 2 2 2
5522024-02-07T18:54:37.870Z19/30 Read block 503 len 4096
5532024-02-07T18:54:37.871Z20/30 Read block 546 len 4096
5542024-02-07T18:54:37.871Z21/30 Flush
5552024-02-07T18:54:37.874Z22/30 Write block 358 len 36864 data: 2 2 2 2 2 2 2 2 2
5562024-02-07T18:54:37.874Z23/30 Read block 512 len 36864
5572024-02-07T18:54:37.877Z24/30 Write block 42 len 40960 data: 2 2 2 2 2 2 2 2 3 4
5582024-02-07T18:54:37.877Z25/30 Flush
5592024-02-07T18:54:37.881Z26/30 Write block 63 len 36864 data: 2 3 3 3 3 3 3 3 2
5602024-02-07T18:54:37.881Z27/30 Read block 419 len 24576
5612024-02-07T18:54:37.881Z28/30 Read block 439 len 36864
5622024-02-07T18:54:37.884Z29/30 Write block 137 len 12288 data: 2 2 2
5632024-02-07T18:54:37.884Z30/30 Write block 316 len 32768 data: 2 2 2 2 3 2 2 2
5642024-02-07T18:54:37.884Z----------------------------------------------------------------
5652024-02-07T18:54:37.884Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:29
5662024-02-07T18:54:37.884ZStates: Active Active Active
5672024-02-07T18:54:37.884ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5682024-02-07T18:54:37.884Z 2 Acked 1001 Write 2 Done Done Done false
5692024-02-07T18:54:37.884Z 3 Acked 1002 Read 4 Done Sent Done false
5702024-02-07T18:54:37.884Z 4 Acked 1003 Write 5 Done Sent Done false
5712024-02-07T18:54:37.885Z 5 Acked 1004 Write 10 Done Sent Done false
5722024-02-07T18:54:37.885Z 6 Acked 1005 Write 8 Done Sent Done false
5732024-02-07T18:54:37.885Z 7 Acked 1006 Flush 0 Done Sent Done false
5742024-02-07T18:54:37.885Z 8 Acked 1007 Write 5 Done Sent Done false
5752024-02-07T18:54:37.885Z 9 Acked 1008 Read 10 Done Sent Done false
5762024-02-07T18:54:37.885Z 10 Acked 1009 Write 4 Done Sent Done false
5772024-02-07T18:54:37.885Z 11 Acked 1010 Flush 0 Done Sent Done false
5782024-02-07T18:54:37.885Z 12 Acked 1011 Write 7 Done Sent Done false
5792024-02-07T18:54:37.885Z 13 Acked 1012 Write 7 Done Sent Done false
5802024-02-07T18:54:37.885Z 14 Acked 1013 Write 3 Done Sent Done false
5812024-02-07T18:54:37.885Z 15 Acked 1014 Read 8 Done Sent Done false
5822024-02-07T18:54:37.885Z 16 Acked 1015 Read 8 Done Sent Done false
5832024-02-07T18:54:37.885Z 17 Acked 1016 Flush 0 Done Sent Done false
5842024-02-07T18:54:37.885Z 18 Acked 1017 Write 8 Done Sent Done false
5852024-02-07T18:54:37.885Z 19 Acked 1018 Read 1 Done Sent Done false
5862024-02-07T18:54:37.885Z 20 Acked 1019 Read 1 Done Sent Done false
5872024-02-07T18:54:37.885Z 21 Acked 1020 Flush 0 Done Sent Done false
5882024-02-07T18:54:37.885Z 22 Acked 1021 Write 9 Done Sent Done false
5892024-02-07T18:54:37.885Z 23 Acked 1022 Read 9 Done Sent Done false
5902024-02-07T18:54:37.885Z 24 Acked 1023 Write 10 Done Sent Done false
5912024-02-07T18:54:37.886Z 25 Acked 1024 Flush 0 Done Sent Done false
5922024-02-07T18:54:37.886Z 26 Acked 1025 Write 9 Done Sent Done false
5932024-02-07T18:54:37.886Z 27 Acked 1026 Read 6 Done Sent Done false
5942024-02-07T18:54:37.886Z 28 Acked 1027 Read 9 Done Sent Done false
5952024-02-07T18:54:37.886Z 29 Acked 1028 Write 3 Done Sent Done false
5962024-02-07T18:54:37.886Z 30 Acked 1029 Write 8 Sent Sent Sent false
5972024-02-07T18:54:37.886Z STATES DS:0 DS:1 DS:2 TOTAL
5982024-02-07T18:54:37.886Z New 0 0 0 0
5992024-02-07T18:54:37.886Z Sent 1 28 1 30
6002024-02-07T18:54:37.886Z Done 28 1 28 57
6012024-02-07T18:54:37.886Z Skipped 0 0 0 0
6022024-02-07T18:54:37.886Z Error 0 0 0 0
6032024-02-07T18:54:37.886ZLast Flush: 1024 1000 1024
6042024-02-07T18:54:37.886ZDownstairs last five completed: 1000
6052024-02-07T18:54:37.886ZUpstairs last five completed: 30 29 28 27 26
6062024-02-07T18:54:37.886Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:37.158882033Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1151}
6072024-02-07T18:54:37.886ZWrote out file "/tmp/test_repair_verify.data"
6082024-02-07T18:54:37.889Z
6092024-02-07T18:54:37.892ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
6102024-02-07T18:54:37.908ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6112024-02-07T18:54:37.908Z 0 000-019 1 1 1 1 1 1 F F F
6122024-02-07T18:54:37.908Z 1 020-039 1 1 1 1 1 1 F F F
6132024-02-07T18:54:37.908Z 2 040-059 4 3 4 12 6 12 F F F <---
6142024-02-07T18:54:37.908Z 3 060-079 4 3 4 10 6 10 T F T <---
6152024-02-07T18:54:37.908Z 4 080-099 4 1 4 8 1 8 F F F <---
6162024-02-07T18:54:37.908Z 5 100-119 4 1 4 9 1 9 F F F <---
6172024-02-07T18:54:37.908Z 6 120-139 4 3 4 9 6 9 T F T <---
6182024-02-07T18:54:37.908Z 7 140-159 3 3 3 6 6 6 F F F
6192024-02-07T18:54:37.908Z 8 160-179 3 3 3 6 6 6 F F F
6202024-02-07T18:54:37.912Z 9 180-199 4 3 4 8 6 8 F F F <---
6212024-02-07T18:54:37.912Z 10 200-219 4 2 4 8 2 8 F F F <---
6222024-02-07T18:54:37.912Z 11 220-239 1 1 1 1 1 1 F F F
6232024-02-07T18:54:37.912Z 12 240-259 1 1 1 1 1 1 F F F
6242024-02-07T18:54:37.912Z 13 260-279 1 1 1 1 1 1 F F F
6252024-02-07T18:54:37.912Z 14 280-299 1 1 1 1 1 1 F F F
6262024-02-07T18:54:37.912Z 15 300-319 4 1 4 8 1 8 T T T <---
6272024-02-07T18:54:37.912Z 16 320-339 2 2 2 4 4 4 T F T <---
6282024-02-07T18:54:37.912Z 17 340-359 4 2 4 12 2 12 F F F <---
6292024-02-07T18:54:37.912Z 18 360-379 4 1 4 12 1 12 F F F <---
6302024-02-07T18:54:37.912Z 19 380-399 4 1 4 11 1 11 F F F <---
6312024-02-07T18:54:37.912Z 20 400-419 1 1 1 1 1 1 F F F
6322024-02-07T18:54:37.912Z 21 420-439 3 3 3 6 6 6 F F F
6332024-02-07T18:54:37.912Z 22 440-459 1 1 1 1 1 1 F F F
6342024-02-07T18:54:37.912Z 23 460-479 1 1 1 1 1 1 F F F
6352024-02-07T18:54:37.912Z 24 480-499 4 1 4 10 1 10 F F F <---
6362024-02-07T18:54:37.913Z 25 500-519 4 1 4 10 1 10 F F F <---
6372024-02-07T18:54:37.913Z 26 520-539 1 1 1 1 1 1 F F F
6382024-02-07T18:54:37.913Z 27 540-559 2 2 2 5 5 5 F F F
6392024-02-07T18:54:37.913Z 28 560-579 1 1 1 1 1 1 F F F
6402024-02-07T18:54:37.913Z 29 580-599 4 1 4 9 1 9 F F F <---
6412024-02-07T18:54:37.913ZMax gen: 4, Max flush: 12
6422024-02-07T18:54:37.913ZError: Difference in extent metadata found!
6432024-02-07T18:54:37.913ZA Difference in extent metadata is expected here
6442024-02-07T18:54:37.913ZOn loop 2
6452024-02-07T18:54:37.913Z
6462024-02-07T18:54:37.916ZVerifying data now
6472024-02-07T18:54:42.627ZNo change after verify
6482024-02-07T18:54:42.627ZLoop: 2 Downstairs dump after verify (and repair):
6492024-02-07T18:54:42.642ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502024-02-07T18:54:42.642Z 0 000-019 1 1 1 1 1 1 F F F
6512024-02-07T18:54:42.642Z 1 020-039 1 1 1 1 1 1 F F F
6522024-02-07T18:54:42.642Z 2 040-059 4 4 4 12 12 12 F F F
6532024-02-07T18:54:42.642Z 3 060-079 5 5 5 13 13 13 F F F
6542024-02-07T18:54:42.642Z 4 080-099 4 4 4 8 8 8 F F F
6552024-02-07T18:54:42.642Z 5 100-119 4 4 4 9 9 9 F F F
6562024-02-07T18:54:42.643Z 6 120-139 5 5 5 13 13 13 F F F
6572024-02-07T18:54:42.644Z 7 140-159 3 3 3 6 6 6 F F F
6582024-02-07T18:54:42.649Z 8 160-179 3 3 3 6 6 6 F F F
6592024-02-07T18:54:42.649Z 9 180-199 4 4 4 8 8 8 F F F
6602024-02-07T18:54:42.650Z 10 200-219 4 4 4 8 8 8 F F F
6612024-02-07T18:54:42.650Z 11 220-239 1 1 1 1 1 1 F F F
6622024-02-07T18:54:42.650Z 12 240-259 1 1 1 1 1 1 F F F
6632024-02-07T18:54:42.650Z 13 260-279 1 1 1 1 1 1 F F F
6642024-02-07T18:54:42.650Z 14 280-299 1 1 1 1 1 1 F F F
6652024-02-07T18:54:42.650Z 15 300-319 5 5 5 13 13 13 F F F
6662024-02-07T18:54:42.650Z 16 320-339 5 5 5 13 13 13 F F F
6672024-02-07T18:54:42.650Z 17 340-359 4 4 4 12 12 12 F F F
6682024-02-07T18:54:42.650Z 18 360-379 4 4 4 12 12 12 F F F
6692024-02-07T18:54:42.650Z 19 380-399 4 4 4 11 11 11 F F F
6702024-02-07T18:54:42.650Z 20 400-419 1 1 1 1 1 1 F F F
6712024-02-07T18:54:42.650Z 21 420-439 3 3 3 6 6 6 F F F
6722024-02-07T18:54:42.650Z 22 440-459 1 1 1 1 1 1 F F F
6732024-02-07T18:54:42.651Z 23 460-479 1 1 1 1 1 1 F F F
6742024-02-07T18:54:42.651Z 24 480-499 4 4 4 10 10 10 F F F
6752024-02-07T18:54:42.651Z 25 500-519 4 4 4 10 10 10 F F F
6762024-02-07T18:54:42.651Z 26 520-539 1 1 1 1 1 1 F F F
6772024-02-07T18:54:42.651Z 27 540-559 2 2 2 5 5 5 F F F
6782024-02-07T18:54:42.651Z 28 560-579 1 1 1 1 1 1 F F F
6792024-02-07T18:54:42.651Z 29 580-599 4 4 4 9 9 9 F F F
6802024-02-07T18:54:42.651ZMax gen: 5, Max flush: 13
6812024-02-07T18:54:42.651Z
6822024-02-07T18:54:42.651ZBegin loop 3
6832024-02-07T18:54:42.651ZDownstairs to restart: 1
6842024-02-07T18:54:42.662Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937084939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159}
6852024-02-07T18:54:42.662Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937359902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159}
6862024-02-07T18:54:42.662Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.93737756Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159}
6872024-02-07T18:54:42.675Z{"msg":"Crucible 1ce3f7f0-1cf9-40ba-8474-733d777fe535 has session id: 8196198a-f4bb-4d94-8545-92958738889a","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937391478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6882024-02-07T18:54:42.675Z{"msg":"Upstairs opts: Upstairs UUID: 1ce3f7f0-1cf9-40ba-8474-733d777fe535, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937430373Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6892024-02-07T18:54:42.675Z{"msg":"Crucible stats registered with UUID: 1ce3f7f0-1cf9-40ba-8474-733d777fe535","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937457769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6902024-02-07T18:54:42.675ZCrucible runtime is spawned
6912024-02-07T18:54:42.675Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937587361Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159}
6922024-02-07T18:54:42.675Z{"msg":"1ce3f7f0-1cf9-40ba-8474-733d777fe535 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937693437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6932024-02-07T18:54:42.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937721933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6942024-02-07T18:54:42.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937766247Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6952024-02-07T18:54:42.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937798653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6962024-02-07T18:54:42.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937834398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6972024-02-07T18:54:42.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937853865Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6982024-02-07T18:54:42.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937884781Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
6992024-02-07T18:54:42.676Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937924916Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7002024-02-07T18:54:42.676Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937954822Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7012024-02-07T18:54:42.676Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.937988957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7022024-02-07T18:54:42.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938018513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7032024-02-07T18:54:42.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938054828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7042024-02-07T18:54:42.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938156175Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7052024-02-07T18:54:42.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938840312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7062024-02-07T18:54:42.676Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938877977Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7072024-02-07T18:54:42.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938894265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7082024-02-07T18:54:42.677Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.938911102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7092024-02-07T18:54:42.677Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.939550976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7102024-02-07T18:54:42.677Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.939603059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7112024-02-07T18:54:42.677Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940100752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7122024-02-07T18:54:42.677Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940139106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7132024-02-07T18:54:42.677Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940165343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7142024-02-07T18:54:42.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940203667Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7152024-02-07T18:54:42.678Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940243032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7162024-02-07T18:54:42.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940286806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7172024-02-07T18:54:42.679Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940319142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7182024-02-07T18:54:42.679Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940353797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7192024-02-07T18:54:42.679Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.940748784Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7202024-02-07T18:54:42.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941211851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7212024-02-07T18:54:42.679Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941248956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7222024-02-07T18:54:42.679Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 12, 13, 8, 9, 13, 6, 6, 8, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941265674Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7232024-02-07T18:54:42.679Z{"msg":"[0]R generation[0..12]: [1, 1, 4, 5, 4, 4, 5, 3, 3, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941287321Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7242024-02-07T18:54:42.679Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941302379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7252024-02-07T18:54:42.680Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 12, 13, 8, 9, 13, 6, 6, 8, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941323216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7262024-02-07T18:54:42.680Z{"msg":"[1]R generation[0..12]: [1, 1, 4, 5, 4, 4, 5, 3, 3, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941337594Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7272024-02-07T18:54:42.680Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941352482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7282024-02-07T18:54:42.680Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 12, 13, 8, 9, 13, 6, 6, 8, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.94136666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7292024-02-07T18:54:42.680Z{"msg":"[2]R generation[0..12]: [1, 1, 4, 5, 4, 4, 5, 3, 3, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941380468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7302024-02-07T18:54:42.680Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
7312024-02-07T18:54:42.680Ztime":"2024-02-07T18:54:41.941403465Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7322024-02-07T18:54:42.680Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941427572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7332024-02-07T18:54:42.680Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.94144171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7342024-02-07T18:54:42.680Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941455398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7352024-02-07T18:54:42.680Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941469336Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7362024-02-07T18:54:42.680Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941495703Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7372024-02-07T18:54:42.680Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941508421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7382024-02-07T18:54:42.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941522469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7392024-02-07T18:54:42.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941542606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7402024-02-07T18:54:42.681Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941556504Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7412024-02-07T18:54:42.681Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941569523Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7422024-02-07T18:54:42.687Z{"msg":"1ce3f7f0-1cf9-40ba-8474-733d777fe535 is now active with session: 8196198a-f4bb-4d94-8545-92958738889a","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941582431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7432024-02-07T18:54:42.687Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941600989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159,"session_id":"8196198a-f4bb-4d94-8545-92958738889a"}
7442024-02-07T18:54:42.687Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:41.941613827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159}
7452024-02-07T18:54:42.687ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7462024-02-07T18:54:42.687ZLoading write count information from file "/tmp/test_repair_verify.data"
7472024-02-07T18:54:42.687ZRun Repair workload
7482024-02-07T18:54:42.687Z01/30 Write block 29 len 36864 data: 2 2 2 2 2 2 2 2 2
7492024-02-07T18:54:42.687Z02/30 Read block 121 len 16384
7502024-02-07T18:54:42.688Z03/30 Write block 145 len 4096 data: 2
7512024-02-07T18:54:42.688Z04/30 Read block 173 len 32768
7522024-02-07T18:54:42.688Z05/30 Read block 273 len 36864
7532024-02-07T18:54:42.688Z06/30 Read block 530 len 4096
7542024-02-07T18:54:42.688Z07/30 Flush
7552024-02-07T18:54:42.688Z08/30 Write block 162 len 40960 data: 4 4 4 4 4 4 4 4 5 5
7562024-02-07T18:54:42.688Z09/30 Read block 508 len 8192
7572024-02-07T18:54:42.688Z10/30 Write block 393 len 32768 data: 2 2 2 2 2 2 2 2
7582024-02-07T18:54:42.688Z11/30 Read block 542 len 28672
7592024-02-07T18:54:42.688Z12/30 Read block 186 len 24576
7602024-02-07T18:54:42.688Z13/30 Write block 339 len 32768 data: 2 2 2 2 2 2 3 3
7612024-02-07T18:54:42.688Z14/30 Write block 411 len 4096 data: 2
7622024-02-07T18:54:42.688Z15/30 Read block 545 len 16384
7632024-02-07T18:54:42.689Z16/30 Read block 22 len 16384
7642024-02-07T18:54:42.689Z17/30 Write block 185 len 4096 data: 2
7652024-02-07T18:54:42.689Z18/30 Flush
7662024-02-07T18:54:42.754Z19/30 Write block 94 len 24576 data: 2 2 2 2 2 2
7672024-02-07T18:54:42.755Z20/30 Read block 106 len 4096
7682024-02-07T18:54:42.755Z21/30 Read block 173 len 12288
7692024-02-07T18:54:42.755Z22/30 Write block 278 len 36864 data: 2 2 2 2 2 2 2 2 2
7702024-02-07T18:54:42.755Z23/30 Read block 594 len 20480
7712024-02-07T18:54:42.755Z24/30 Read block 262 len 16384
7722024-02-07T18:54:42.755Z25/30 Flush
7732024-02-07T18:54:42.755Z26/30 Write block 344 len 32768 data: 3 4 4 2 2 2 2 2
7742024-02-07T18:54:42.755Z27/30 Read block 348 len 28672
7752024-02-07T18:54:42.756Z28/30 Read block 352 len 28672
7762024-02-07T18:54:42.756Z29/30 Write block 60 len 40960 data: 2 2 2 3 4 4 4 4 4 4
7772024-02-07T18:54:42.756Z30/30 Read block 227 len 40960
7782024-02-07T18:54:42.756Z----------------------------------------------------------------
7792024-02-07T18:54:42.756Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7802024-02-07T18:54:42.756ZStates: Active Active Active
7812024-02-07T18:54:42.756ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7822024-02-07T18:54:42.756Z 1 Acked 1000 Write 9 Done Done Done false
7832024-02-07T18:54:42.756Z 2 Acked 1001 Read 4 Done Done Done false
7842024-02-07T18:54:42.756Z 3 Acked 1002 Write 1 Done Done Done false
7852024-02-07T18:54:42.756Z 4 Acked 1003 Read 8 Done Sent Done false
7862024-02-07T18:54:42.757Z 5 Acked 1004 Read 9 Done Sent Done false
7872024-02-07T18:54:42.757Z 6 Acked 1005 Read 1 Done Sent Done false
7882024-02-07T18:54:42.757Z 7 Acked 1006 Flush 0 Done Sent Done false
7892024-02-07T18:54:42.757Z 8 Acked 1007 Write 10 Done Sent Done false
7902024-02-07T18:54:42.757Z 9 Acked 1008 Read 2 Done Sent Done false
7912024-02-07T18:54:42.757Z 10 Acked 1009 Write 8 Done Sent Done false
7922024-02-07T18:54:42.757Z 11 Acked 1010 Read 7 Done Sent Done false
7932024-02-07T18:54:42.757Z 12 Acked 1011 Read 6 Done Sent Done false
7942024-02-07T18:54:42.757Z 13 Acked 1012 Write 8 Done Sent Done false
7952024-02-07T18:54:42.757Z 14 Acked 1013 Write 1 Done Sent Done false
7962024-02-07T18:54:42.757Z 15 Acked 1014 Read 4 Done Sent Done false
7972024-02-07T18:54:42.758Z 16 Acked 1015 Read 4 Done Sent Done false
7982024-02-07T18:54:42.758Z 17 Acked 1016 Write 1 Done Sent Done false
7992024-02-07T18:54:42.758Z 18 Acked 1017 Flush 0 Done Sent Done false
8002024-02-07T18:54:42.758Z 19 Acked 1018 Write 6 Done Sent Done false
8012024-02-07T18:54:42.758Z 20 Acked 1019 Read 1 Done Sent Done false
8022024-02-07T18:54:42.758Z 21 Acked 1020 Read 3 Done Sent Done false
8032024-02-07T18:54:42.758Z 22 Acked 1021 Write 9 Done Sent Done false
8042024-02-07T18:54:42.758Z 23 Acked 1022 Read 5 Done Sent Done false
8052024-02-07T18:54:42.758Z 24 Acked 1023 Read 4 Done Sent Done false
8062024-02-07T18:54:42.758Z 25 Acked 1024 Flush 0 Done Sent Done false
8072024-02-07T18:54:42.759Z 26 Acked 1025 Write 8 Done Sent Done false
8082024-02-07T18:54:42.759Z 27 Acked 1026 Read 7 Done Sent Done false
8092024-02-07T18:54:42.759Z 28 Acked 1027 Read 7 Done Sent Done false
8102024-02-07T18:54:42.759Z 29 Acked 1028 Write 10 Done Sent Done false
8112024-02-07T18:54:42.759Z 30 Acked 1029 Read 10 Sent Sent Done false
8122024-02-07T18:54:42.759Z STATES DS:0 DS:1 DS:2 TOTAL
8132024-02-07T18:54:42.759Z New 0 0 0 0
8142024-02-07T18:54:42.759Z Sent 1 27 0 28
8152024-02-07T18:54:42.759Z Done 29 3 30 62
8162024-02-07T18:54:42.759Z Skipped 0 0 0 0
8172024-02-07T18:54:42.759Z Error 0 0 0 0
8182024-02-07T18:54:42.760ZLast Flush: 1024 0 1024
8192024-02-07T18:54:42.760ZDownstairs last five completed:
8202024-02-07T18:54:42.760ZUpstairs last five completed: 30 29 28 27 26
8212024-02-07T18:54:42.760Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:41.974460794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1159}
8222024-02-07T18:54:42.760ZWrote out file "/tmp/test_repair_verify.data"
8232024-02-07T18:54:42.760Z
8242024-02-07T18:54:42.760ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8252024-02-07T18:54:42.760ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8262024-02-07T18:54:42.760Z 0 000-019 1 1 1 1 1 1 F F F
8272024-02-07T18:54:42.760Z 1 020-039 6 1 6 14 1 14 F T F <---
8282024-02-07T18:54:42.760Z 2 040-059 4 4 4 12 12 12 F F F
8292024-02-07T18:54:42.760Z 3 060-079 5 5 5 13 13 13 T F T <---
8302024-02-07T18:54:42.761Z 4 080-099 6 4 6 16 8 16 F F F <---
8312024-02-07T18:54:42.761Z 5 100-119 4 4 4 9 9 9 F F F
8322024-02-07T18:54:42.761Z 6 120-139 5 5 5 13 13 13 F F F
8332024-02-07T18:54:42.761Z 7 140-159 6 3 6 14 6 14 F T F <---
8342024-02-07T18:54:42.761Z 8 160-179 6 3 6 15 6 15 F F F <---
8352024-02-07T18:54:42.761Z 9 180-199 6 4 6 15 8 15 F F F <---
8362024-02-07T18:54:42.761Z 10 200-219 4 4 4 8 8 8 F F F
8372024-02-07T18:54:42.761Z 11 220-239 1 1 1 1 1 1 F F F
8382024-02-07T18:54:42.762Z 12 240-259 1 1 1 1 1 1 F F F
8392024-02-07T18:54:42.762Z 13 260-279 6 1 6 16 1 16 F F F <---
8402024-02-07T18:54:42.762Z 14 280-299 6 1 6 16 1 16 F F F <---
8412024-02-07T18:54:42.762Z 15 300-319 5 5 5 13 13 13 F F F
8422024-02-07T18:54:42.762Z 16 320-339 6 5 6 15 13 15 F F F <---
8432024-02-07T18:54:42.762Z 17 340-359 6 4 6 15 12 15 T F T <---
8442024-02-07T18:54:42.762Z 18 360-379 4 4 4 12 12 12 F F F
8452024-02-07T18:54:42.762Z 19 380-399 6 4 6 15 11 15 F F F <---
8462024-02-07T18:54:42.762Z 20 400-419 6 1 6 15 1 15 F F F <---
8472024-02-07T18:54:42.762Z 21 420-439 3 3 3 6 6 6 F F F
8482024-02-07T18:54:42.762Z 22 440-459 1 1 1 1 1 1 F F F
8492024-02-07T18:54:42.762Z 23 460-479 1 1 1 1 1 1 F F F
8502024-02-07T18:54:42.762Z 24 480-499 4 4 4 10 10 10 F F F
8512024-02-07T18:54:42.763Z 25 500-519 4 4 4 10 10 10 F F F
8522024-02-07T18:54:42.763Z 26 520-539 1 1 1 1 1 1 F F F
8532024-02-07T18:54:42.763Z 27 540-559 2 2 2 5 5 5 F F F
8542024-02-07T18:54:42.763Z 28 560-579 1 1 1 1 1 1 F F F
8552024-02-07T18:54:42.763Z 29 580-599 4 4 4 9 9 9 F F F
8562024-02-07T18:54:42.763ZMax gen: 6, Max flush: 16
8572024-02-07T18:54:42.771ZError: Difference in extent metadata found!
8582024-02-07T18:54:42.771ZA Difference in extent metadata is expected here
8592024-02-07T18:54:42.771ZOn loop 3
8602024-02-07T18:54:42.771Z
8612024-02-07T18:54:42.771ZVerifying data now
8622024-02-07T18:54:48.309ZNo change after verify
8632024-02-07T18:54:48.309ZLoop: 3 Downstairs dump after verify (and repair):
8642024-02-07T18:54:48.325ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8652024-02-07T18:54:48.326Z 0 000-019 1 1 1 1 1 1 F F F
8662024-02-07T18:54:48.326Z 1 020-039 6 6 6 14 14 14 F F F
8672024-02-07T18:54:48.326Z 2 040-059 4 4 4 12 12 12 F F F
8682024-02-07T18:54:48.326Z 3 060-079 7 7 7 17 17 17 F F F
8692024-02-07T18:54:48.326Z 4 080-099 6 6 6 16 16 16 F F F
8702024-02-07T18:54:48.326Z 5 100-119 4 4 4 9 9 9 F F F
8712024-02-07T18:54:48.326Z 6 120-139 5 5 5 13 13 13 F F F
8722024-02-07T18:54:48.326Z 7 140-159 6 6 6 14 14 14 F F F
8732024-02-07T18:54:48.326Z 8 160-179 6 6 6 15 15 15 F F F
8742024-02-07T18:54:48.326Z 9 180-199 6 6 6 15 15 15 F F F
8752024-02-07T18:54:48.334Z 10 200-219 4 4 4 8 8 8 F F F
8762024-02-07T18:54:48.334Z 11 220-239 1 1 1 1 1 1 F F F
8772024-02-07T18:54:48.334Z 12 240-259 1 1 1 1 1 1 F F F
8782024-02-07T18:54:48.334Z 13 260-279 6 6 6 16 16 16 F F F
8792024-02-07T18:54:48.334Z 14 280-299 6 6 6 16 16 16 F F F
8802024-02-07T18:54:48.334Z 15 300-319 5 5 5 13 13 13 F F F
8812024-02-07T18:54:48.334Z 16 320-339 6 6 6 15 15 15 F F F
8822024-02-07T18:54:48.334Z 17 340-359 7 7 7 17 17 17 F F F
8832024-02-07T18:54:48.334Z 18 360-379 4 4 4 12 12 12 F F F
8842024-02-07T18:54:48.335Z 19 380-399 6 6 6 15 15 15 F F F
8852024-02-07T18:54:48.335Z 20 400-419 6 6 6 15 15 15 F F F
8862024-02-07T18:54:48.335Z 21 420-439 3 3 3 6 6 6 F F F
8872024-02-07T18:54:48.335Z 22 440-459 1 1 1 1 1 1 F F F
8882024-02-07T18:54:48.335Z 23 460-479 1 1 1 1 1 1 F F F
8892024-02-07T18:54:48.335Z 24 480-499 4 4 4 10 10 10 F F F
8902024-02-07T18:54:48.335Z 25 500-519 4 4 4 10 10 10 F F F
8912024-02-07T18:54:48.335Z 26 520-539 1 1 1 1 1 1 F F F
8922024-02-07T18:54:48.335Z 27 540-559 2 2 2 5 5 5 F F F
8932024-02-07T18:54:48.335Z 28 560-579 1 1 1 1 1 1 F F F
8942024-02-07T18:54:48.335Z 29 580-599 4 4 4 9 9 9 F F F
8952024-02-07T18:54:48.335ZMax gen: 7, Max flush: 17
8962024-02-07T18:54:48.335Z
8972024-02-07T18:54:48.335ZBegin loop 4
8982024-02-07T18:54:48.335ZDownstairs to restart: 2
8992024-02-07T18:54:48.380Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.620763469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167}
9002024-02-07T18:54:48.380Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621025893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167}
9012024-02-07T18:54:48.380Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.62105092Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167}
9022024-02-07T18:54:48.380Z{Crucible runtime is spawned
9032024-02-07T18:54:48.380Z"msg":"Crucible 0ef6cf71-8641-4e12-a3cb-8f15a2e5980d has session id: 7bb830dc-6be5-4a53-921d-51a78c9ee9fb","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621117121Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9042024-02-07T18:54:48.380Z{"msg":"Upstairs opts: Upstairs UUID: 0ef6cf71-8641-4e12-a3cb-8f15a2e5980d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621176333Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9052024-02-07T18:54:48.380Z{"msg":"Crucible stats registered with UUID: 0ef6cf71-8641-4e12-a3cb-8f15a2e5980d","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621208359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9062024-02-07T18:54:48.380Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621225286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167}
9072024-02-07T18:54:48.380Z{"msg":"0ef6cf71-8641-4e12-a3cb-8f15a2e5980d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621338241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9082024-02-07T18:54:48.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621370947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9092024-02-07T18:54:48.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621390424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9102024-02-07T18:54:48.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621408622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9112024-02-07T18:54:48.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621425759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9122024-02-07T18:54:48.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621452176Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9132024-02-07T18:54:48.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621479482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9142024-02-07T18:54:48.380Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621611164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9152024-02-07T18:54:48.380Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621681305Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9162024-02-07T18:54:48.380Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621731218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9172024-02-07T18:54:48.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621769633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9182024-02-07T18:54:48.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.621796519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9192024-02-07T18:54:48.381Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:47.621898975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9202024-02-07T18:54:48.381Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:47.62194023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9212024-02-07T18:54:48.381Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:47.621980534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9222024-02-07T18:54:48.381Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622006771Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9232024-02-07T18:54:48.381Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622036497Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9242024-02-07T18:54:48.381Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622178857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9252024-02-07T18:54:48.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622335836Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9262024-02-07T18:54:48.381Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622366422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9272024-02-07T18:54:48.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622385629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9282024-02-07T18:54:48.381Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622463789Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9292024-02-07T18:54:48.381Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.622926307Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9302024-02-07T18:54:48.381Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.623070767Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9312024-02-07T18:54:48.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.623530575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9322024-02-07T18:54:48.381Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.623582408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9332024-02-07T18:54:48.382Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.623644979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9342024-02-07T18:54:48.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.623674555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9352024-02-07T18:54:48.382Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.623689843Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9362024-02-07T18:54:48.382Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:47.6237117Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9372024-02-07T18:54:49.355Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.623395192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9382024-02-07T18:54:49.355Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.623519235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9392024-02-07T18:54:49.385Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.624304519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9402024-02-07T18:54:49.385Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.624351453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9412024-02-07T18:54:49.385Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.624880151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9422024-02-07T18:54:49.385Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625528413Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9432024-02-07T18:54:49.386Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625577237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9442024-02-07T18:54:49.386Z{"msg":"[0]R flush_numbers[0..12]: [1, 14, 12, 17, 16, 9, 13, 14, 15, 15, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625615832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9452024-02-07T18:54:49.386Z{"msg":"[0]R generation[0..12]: [1, 6, 4, 7, 6, 4, 5, 6, 6, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625652077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9462024-02-07T18:54:49.386Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625689012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9472024-02-07T18:54:49.386ZWait for a query_work_queue command to finish before sending IO
9482024-02-07T18:54:49.386Z{"msg":"[1]R flush_numbers[0..12]: [1, 14, 12, 17, 16, 9, 13, 14, 15, 15, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625736015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9492024-02-07T18:54:49.386Z{"msg":"[1]R generation[0..12]: [1, 6, 4, 7, 6, 4, 5, 6, 6, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625770171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9502024-02-07T18:54:49.386Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625803786Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9512024-02-07T18:54:49.386Z{"msg":"[2]R flush_numbers[0..12]: [1, 14, 12, 17, 16, 9, 13, 14, 15, 15, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.62585156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9522024-02-07T18:54:49.386Z{"msg":"[2]R generation[0..12]: [1, 6, 4, 7, 6, 4, 5, 6, 6, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625937348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9532024-02-07T18:54:49.386Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.625972033Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9542024-02-07T18:54:49.386Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626045983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9552024-02-07T18:54:49.386Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.62607579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9562024-02-07T18:54:49.386Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626110835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9572024-02-07T18:54:49.386Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.62614725Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9582024-02-07T18:54:49.386Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626175386Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9592024-02-07T18:54:49.386Z,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9602024-02-07T18:54:49.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626212211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9612024-02-07T18:54:49.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626233808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9622024-02-07T18:54:49.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626248556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9632024-02-07T18:54:49.386Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626270343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9642024-02-07T18:54:49.387Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626284281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9652024-02-07T18:54:49.387Z{"msg":"0ef6cf71-8641-4e12-a3cb-8f15a2e5980d is now active with session: 7bb830dc-6be5-4a53-921d-51a78c9ee9fb","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626297269Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9662024-02-07T18:54:49.387Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626317877Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167,"session_id":"7bb830dc-6be5-4a53-921d-51a78c9ee9fb"}
9672024-02-07T18:54:49.387Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:48.626331155Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167}
9682024-02-07T18:54:49.387ZLoading write count information from file "/tmp/test_repair_verify.data"
9692024-02-07T18:54:49.387ZRun Repair workload
9702024-02-07T18:54:49.387Z01/30 Flush
9712024-02-07T18:54:49.387Z02/30 Write block 202 len 20480 data: 2 3 3 3 3
9722024-02-07T18:54:49.387Z03/30 Read block 543 len 8192
9732024-02-07T18:54:49.387Z04/30 Read block 394 len 12288
9742024-02-07T18:54:49.387Z05/30 Write block 232 len 16384 data: 2 2 2 2
9752024-02-07T18:54:49.387Z06/30 Write block 53 len 28672 data: 4 4 3 3 3 3 3
9762024-02-07T18:54:49.387Z07/30 Read block 577 len 28672
9772024-02-07T18:54:49.387Z08/30 Flush
9782024-02-07T18:54:49.387Z09/30 Write block 236 len 16384 data: 2 2 2 2
9792024-02-07T18:54:49.387Z10/30 Write block 119 len 4096 data: 3
9802024-02-07T18:54:49.387Z11/30 Write block 350 len 12288 data: 3 3 2
9812024-02-07T18:54:49.387Z12/30 Read block 285 len 40960
9822024-02-07T18:54:49.388Z13/30 Write block 417 len 28672 data: 2 2 2 2 2 2 2
9832024-02-07T18:54:49.388Z14/30 Read block 387 len 40960
9842024-02-07T18:54:49.388Z15/30 Read block 221 len 4096
9852024-02-07T18:54:49.388Z16/30 Read block 330 len 28672
9862024-02-07T18:54:49.388Z17/30 Read block 395 len 12288
9872024-02-07T18:54:49.388Z18/30 Write block 463 len 16384 data: 2 2 2 2
9882024-02-07T18:54:49.388Z19/30 Write block 453 len 4096 data: 2
9892024-02-07T18:54:49.388Z20/30 Read block 325 len 8192
9902024-02-07T18:54:49.448Z21/30 Read block 355 len 20480
9912024-02-07T18:54:49.448Z22/30 Write block 424 len 20480 data: 2 2 2 2 2
9922024-02-07T18:54:49.449Z23/30 Write block 552 len 36864 data: 2 2 2 2 2 2 2 2 2
9932024-02-07T18:54:49.449Z24/30 Read block 46 len 16384
9942024-02-07T18:54:49.449Z25/30 Read block 419 len 20480
9952024-02-07T18:54:49.449Z26/30 Read block 71 len 36864
9962024-02-07T18:54:49.449Z27/30 Read block 62 len 16384
9972024-02-07T18:54:49.449Z28/30 Read block 385 len 24576
9982024-02-07T18:54:49.449Z29/30 Write block 395 len 40960 data: 3 3 3 3 3 3 2 2 2 2
9992024-02-07T18:54:49.449Z30/30 Write block 179 len 20480 data: 2 2 2 2 2
10002024-02-07T18:54:49.449Z----------------------------------------------------------------
10012024-02-07T18:54:49.449Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:29
10022024-02-07T18:54:49.449ZStates: Active Active Active
10032024-02-07T18:54:49.450ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
10042024-02-07T18:54:49.450Z 2 Acked 1001 Write 5 Done Done Done false
10052024-02-07T18:54:49.450Z 3 Acked 1002 Read 2 Done Done Done false
10062024-02-07T18:54:49.450Z 4 Acked 1003 Read 3 Done Done Sent false
10072024-02-07T18:54:49.450Z 5 Acked 1004 Write 4 Done Done Sent false
10082024-02-07T18:54:49.450Z 6 Acked 1005 Write 7 Done Done Sent false
10092024-02-07T18:54:49.450Z 7 Acked 1006 Read 7 Done Done Sent false
10102024-02-07T18:54:49.450Z 8 Acked 1007 Flush 0 Done Done Sent false
10112024-02-07T18:54:49.450Z 9 Acked 1008 Write 4 Done Done Sent false
10122024-02-07T18:54:49.450Z 10 Acked 1009 Write 1 Done Done Sent false
10132024-02-07T18:54:49.450Z 11 Acked 1010 Write 3 Done Done Sent false
10142024-02-07T18:54:49.450Z 12 Acked 1011 Read 10 Done Done Sent false
10152024-02-07T18:54:49.451Z 13 Acked 1012 Write 7 Done Done Sent false
10162024-02-07T18:54:49.451Z 14 Acked 1013 Read 10 Done Done Sent false
10172024-02-07T18:54:49.451Z 15 Acked 1014 Read 1 Done Done Sent false
10182024-02-07T18:54:49.451Z 16 Acked 1015 Read 7 Done Done Sent false
10192024-02-07T18:54:49.451Z 17 Acked 1016 Read 3 Done Done Sent false
10202024-02-07T18:54:49.451Z 18 Acked 1017 Write 4 Done Done Sent false
10212024-02-07T18:54:49.451Z 19 Acked 1018 Write 1 Done Done Sent false
10222024-02-07T18:54:49.451Z 20 Acked 1019 Read 2 Done Done Sent false
10232024-02-07T18:54:49.451Z 21 Acked 1020 Read 5 Done Done Sent false
10242024-02-07T18:54:49.451Z 22 Acked 1021 Write 5 Done Done Sent false
10252024-02-07T18:54:49.451Z 23 Acked 1022 Write 9 Done Done Sent false
10262024-02-07T18:54:49.451Z 24 Acked 1023 Read 4 Done Done Sent false
10272024-02-07T18:54:49.452Z 25 Acked 1024 Read 5 Done Done Sent false
10282024-02-07T18:54:49.452Z 26 Acked 1025 Read 9 Done Done Sent false
10292024-02-07T18:54:49.452Z 27 Acked 1026 Read 4 Done Done Sent false
10302024-02-07T18:54:49.452Z 28 Acked 1027 Read 6 Done Done Sent false
10312024-02-07T18:54:49.452Z 29 Acked 1028 Write 10 Sent Sent Sent false
10322024-02-07T18:54:49.452Z 30 Acked 1029 Write 5 Sent Sent Sent false
10332024-02-07T18:54:49.452Z STATES DS:0 DS:1 DS:2 TOTAL
10342024-02-07T18:54:49.452Z New 0 0 0 0
10352024-02-07T18:54:49.452Z Sent 2 2 27 31
10362024-02-07T18:54:49.452Z Done 27 27 2 56
10372024-02-07T18:54:49.452Z Skipped 0 0 0 0
10382024-02-07T18:54:49.452Z Error 0 0 0 0
10392024-02-07T18:54:49.452ZLast Flush: 1007 1007 1000
10402024-02-07T18:54:49.452ZDownstairs last five completed: 1000
10412024-02-07T18:54:49.453ZUpstairs last five completed: 30 29 28 27 26
10422024-02-07T18:54:49.453Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:48.653261793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1167}
10432024-02-07T18:54:49.453ZWrote out file "/tmp/test_repair_verify.data"
10442024-02-07T18:54:49.453Z
10452024-02-07T18:54:49.453ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10462024-02-07T18:54:49.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10472024-02-07T18:54:49.453Z 0 000-019 1 1 1 1 1 1 F F F
10482024-02-07T18:54:49.453Z 1 020-039 6 6 6 14 14 14 F F F
10492024-02-07T18:54:49.453Z 2 040-059 8 8 4 19 19 12 F F F <---
10502024-02-07T18:54:49.453Z 3 060-079 7 7 7 17 17 17 F F F
10512024-02-07T18:54:49.453Z 4 080-099 6 6 6 16 16 16 F F F
10522024-02-07T18:54:49.453Z 5 100-119 4 4 4 9 9 9 T T F <---
10532024-02-07T18:54:49.453Z 6 120-139 5 5 5 13 13 13 F F F
10542024-02-07T18:54:49.453Z 7 140-159 6 6 6 14 14 14 F F F
10552024-02-07T18:54:49.453Z 8 160-179 6 6 6 15 15 15 T T F <---
10562024-02-07T18:54:49.453Z 9 180-199 6 6 6 15 15 15 T T F <---
10572024-02-07T18:54:49.453Z 10 200-219 8 8 4 19 19 8 F F T <---
10582024-02-07T18:54:49.453Z 11 220-239 8 8 1 19 19 1 T T F <---
10592024-02-07T18:54:49.454Z 12 240-259 1 1 1 1 1 1 F F F
10602024-02-07T18:54:49.454Z 13 260-279 6 6 6 16 16 16 F F F
10612024-02-07T18:54:49.454Z 14 280-299 6 6 6 16 16 16 F F F
10622024-02-07T18:54:49.454Z 15 300-319 5 5 5 13 13 13 F F F
10632024-02-07T18:54:49.454Z 16 320-339 6 6 6 15 15 15 F F F
10642024-02-07T18:54:49.454Z 17 340-359 7 7 7 17 17 17 T T F <---
10652024-02-07T18:54:49.454Z 18 360-379 4 4 4 12 12 12 F F F
10662024-02-07T18:54:49.454Z 19 380-399 6 6 6 15 15 15 T T F <---
10672024-02-07T18:54:49.454Z 20 400-419 6 6 6 15 15 15 T T F <---
10682024-02-07T18:54:49.454Z 21 420-439 3 3 3 6 6 6 T T F <---
10692024-02-07T18:54:49.454Z 22 440-459 1 1 1 1 1 1 T T F <---
10702024-02-07T18:54:49.454Z 23 460-479 1 1 1 1 1 1 T T F <---
10712024-02-07T18:54:49.454Z 24 480-499 4 4 4 10 10 10 F F F
10722024-02-07T18:54:49.455Z 25 500-519 4 4 4 10 10 10 F F F
10732024-02-07T18:54:49.455Z 26 520-539 1 1 1 1 1 1 F F F
10742024-02-07T18:54:49.455Z 27 540-559 2 2 2 5 5 5 T T F <---
10752024-02-07T18:54:49.455Z 28 560-579 1 1 1 1 1 1 T T F <---
10762024-02-07T18:54:49.455Z 29 580-599 4 4 4 9 9 9 F F F
10772024-02-07T18:54:49.455ZMax gen: 8, Max flush: 19
10782024-02-07T18:54:49.455ZError: Difference in extent metadata found!
10792024-02-07T18:54:49.455ZA Difference in extent metadata is expected here
10802024-02-07T18:54:49.455ZOn loop 4
10812024-02-07T18:54:49.455Z
10822024-02-07T18:54:49.455ZVerifying data now
10832024-02-07T18:54:56.387ZNo change after verify
10842024-02-07T18:54:56.387ZLoop: 4 Downstairs dump after verify (and repair):
10852024-02-07T18:54:56.388ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10862024-02-07T18:54:56.388Z 0 000-019 1 1 1 1 1 1 F F F
10872024-02-07T18:54:56.388Z 1 020-039 6 6 6 14 14 14 F F F
10882024-02-07T18:54:56.388Z 2 040-059 8 8 8 19 19 19 F F F
10892024-02-07T18:54:56.388Z 3 060-079 7 7 7 17 17 17 F F F
10902024-02-07T18:54:56.388Z 4 080-099 6 6 6 16 16 16 F F F
10912024-02-07T18:54:56.388Z 5 100-119 9 9 9 20 20 20 F F F
10922024-02-07T18:54:56.388Z 6 120-139 5 5 5 13 13 13 F F F
10932024-02-07T18:54:56.388Z 7 140-159 6 6 6 14 14 14 F F F
10942024-02-07T18:54:56.388Z 8 160-179 9 9 9 20 20 20 F F F
10952024-02-07T18:54:56.388Z 9 180-199 9 9 9 20 20 20 F F F
10962024-02-07T18:54:56.388Z 10 200-219 8 8 8 19 19 19 F F F
10972024-02-07T18:54:56.388Z 11 220-239 9 9 9 20 20 20 F F F
10982024-02-07T18:54:56.388Z 12 240-259 1 1 1 1 1 1 F F F
10992024-02-07T18:54:56.388Z 13 260-279 6 6 6 16 16 16 F F F
11002024-02-07T18:54:56.388Z 14 280-299 6 6 6 16 16 16 F F F
11012024-02-07T18:54:56.389Z 15 300-319 5 5 5 13 13 13 F F F
11022024-02-07T18:54:56.389Z 16 320-339 6 6 6 15 15 15 F F F
11032024-02-07T18:54:56.389Z 17 340-359 9 9 9 20 20 20 F F F
11042024-02-07T18:54:56.389Z 18 360-379 4 4 4 12 12 12 F F F
11052024-02-07T18:54:56.389Z 19 380-399 9 9 9 20 20 20 F F F
11062024-02-07T18:54:56.389Z 20 400-419 9 9 9 20 20 20 F F F
11072024-02-07T18:54:56.389Z 21 420-439 9 9 9 20 20 20 F F F
11082024-02-07T18:54:56.389Z 22 440-459 9 9 9 20 20 20 F F F
11092024-02-07T18:54:56.837Z 23 460-479 9 9 9 20 20 20 F F F
11102024-02-07T18:54:56.837Z 24 480-499 4 4 4 10 10 10 F F F
11112024-02-07T18:54:56.837Z 25 500-519 4 4 4 10 10 10 F F F
11122024-02-07T18:54:56.837Z 26 520-539 1 1 1 1 1 1 F F F
11132024-02-07T18:54:56.837Z 27 540-559 9 9 9 20 20 20 F F F
11142024-02-07T18:54:56.837Z 28 560-579 9 9 9 20 20 20 F F F
11152024-02-07T18:54:56.837Z 29 580-599 4 4 4 9 9 9 F F F
11162024-02-07T18:54:56.838ZMax gen: 9, Max flush: 20
11172024-02-07T18:54:56.838Z
11182024-02-07T18:54:56.838ZBegin loop 5
11192024-02-07T18:54:56.838ZDownstairs to restart: 2
11202024-02-07T18:54:56.838Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402256249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175}
11212024-02-07T18:54:56.838Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402487088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175}
11222024-02-07T18:54:56.838Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402524422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175}
11232024-02-07T18:54:56.838Z{"msg":"Crucible 4b2a23da-5907-46ea-8859-113893572f2a has session id: 64464f70-20b2-4b9d-a3ae-a5e9883a7c18","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402560798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175Crucible runtime is spawned
11242024-02-07T18:54:56.838Z,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11252024-02-07T18:54:56.838Z{"msg":"Upstairs opts: Upstairs UUID: 4b2a23da-5907-46ea-8859-113893572f2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402610521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11262024-02-07T18:54:56.838Z{"msg":"Crucible stats registered with UUID: 4b2a23da-5907-46ea-8859-113893572f2a","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402655755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11272024-02-07T18:54:56.838Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40269206Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175}
11282024-02-07T18:54:56.838Z{"msg":"4b2a23da-5907-46ea-8859-113893572f2a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40276328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11292024-02-07T18:54:56.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402807634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11302024-02-07T18:54:56.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40284214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11312024-02-07T18:54:56.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402859397Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11322024-02-07T18:54:56.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402883974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11332024-02-07T18:54:56.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402921969Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11342024-02-07T18:54:56.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402955024Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11352024-02-07T18:54:56.838Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.402982171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11362024-02-07T18:54:56.838Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403019286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11372024-02-07T18:54:56.839Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403055491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11382024-02-07T18:54:56.839Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403092266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11392024-02-07T18:54:56.839Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403120132Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11402024-02-07T18:54:56.839Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403163796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11412024-02-07T18:54:56.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403763455Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11422024-02-07T18:54:56.839Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40380274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11432024-02-07T18:54:56.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403831066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11442024-02-07T18:54:56.839Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.403867281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11452024-02-07T18:54:56.839Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.404418926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11462024-02-07T18:54:56.839Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40446806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11472024-02-07T18:54:56.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405039942Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11482024-02-07T18:54:56.839Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405081337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11492024-02-07T18:54:56.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405124411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11502024-02-07T18:54:56.839Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405163496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11512024-02-07T18:54:56.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405200341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11522024-02-07T18:54:56.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405231516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11532024-02-07T18:54:56.839Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405264762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11542024-02-07T18:54:56.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405297538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11552024-02-07T18:54:56.839Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.405627853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11562024-02-07T18:54:56.840Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406068933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11572024-02-07T18:54:56.840Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406109648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11582024-02-07T18:54:56.840Z{"msg":"[0]R flush_numbers[0..12]: [1, 14, 19, 17, 16, 20, 13, 14, 20, 20, 19, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406133754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11592024-02-07T18:54:56.840Z{"msg":"[0]R generation[0..12]: [1, 6, 8, 7, 6, 9, 5, 6, 9, 9, 8, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406149192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11602024-02-07T18:54:56.840Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40616417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11612024-02-07T18:54:56.840Z{"msg":"[1]R flush_numbers[0..12]: [1, 14, 19, 17, 16, 20, 13, 14, 20, 20, 19, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406185318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11622024-02-07T18:54:56.840Z{"msg":"[1]R generation[0..12]: [1, 6, 8, 7, 6, 9, 5, 6, 9, 9, 8, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406212114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11632024-02-07T18:54:56.840ZWait for a query_work_queue command to finish before sending IO
11642024-02-07T18:54:56.840Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406252688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11652024-02-07T18:54:56.840Z{"msg":"[2]R flush_numbers[0..12]: [1, 14, 19, 17, 16, 20, 13, 14, 20, 20, 19, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406289483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11662024-02-07T18:54:56.840Z{"msg":"[2]R generation[0..12]: [1, 6, 8, 7, 6, 9, 5, 6, 9, 9, 8, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406322649Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11672024-02-07T18:54:56.840Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406359604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11682024-02-07T18:54:56.840Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40638925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11692024-02-07T18:54:56.840Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406424805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11702024-02-07T18:54:56.840Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.40646209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11712024-02-07T18:54:56.840Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406494636Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11722024-02-07T18:54:56.840Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406522432Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11732024-02-07T18:54:56.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406555917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11742024-02-07T18:54:56.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406592732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11752024-02-07T18:54:56.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11762024-02-07T18:54:56.840Z,"time":"2024-02-07T18:54:53.406627768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11772024-02-07T18:54:56.840Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406661133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11782024-02-07T18:54:56.841Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406689399Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11792024-02-07T18:54:56.841Z{"msg":"4b2a23da-5907-46ea-8859-113893572f2a is now active with session: 64464f70-20b2-4b9d-a3ae-a5e9883a7c18","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406710716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11802024-02-07T18:54:56.841Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406724454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175,"session_id":"64464f70-20b2-4b9d-a3ae-a5e9883a7c18"}
11812024-02-07T18:54:56.841Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:53.406743512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175}
11822024-02-07T18:54:56.841ZLoading write count information from file "/tmp/test_repair_verify.data"
11832024-02-07T18:54:56.841ZError: Difference in extent metadata found!
11842024-02-07T18:54:56.841ZRun Repair workload
11852024-02-07T18:54:56.841Z01/30 Write block 358 len 20480 data: 3 3 3 3 3
11862024-02-07T18:54:56.841Z02/30 Write block 315 len 24576 data: 2 3 3 3 3 4
11872024-02-07T18:54:56.841Z03/30 Read block 242 len 8192
11882024-02-07T18:54:56.841Z04/30 Read block 484 len 36864
11892024-02-07T18:54:56.841Z05/30 Read block 335 len 20480
11902024-02-07T18:54:56.841Z06/30 Read block 80 len 4096
11912024-02-07T18:54:56.841Z07/30 Write block 247 len 8192 data: 2 2
11922024-02-07T18:54:56.841Z08/30 Read block 65 len 36864
11932024-02-07T18:54:56.841Z09/30 Write block 40 len 16384 data: 2 2 3 3
11942024-02-07T18:54:56.841Z10/30 Read block 558 len 8192
11952024-02-07T18:54:56.841Z11/30 Read block 128 len 20480
11962024-02-07T18:54:56.841Z12/30 Read block 429 len 20480
11972024-02-07T18:54:56.841Z13/30 Read block 468 len 36864
11982024-02-07T18:54:56.842Z14/30 Read block 202 len 28672
11992024-02-07T18:54:56.842Z15/30 Write block 556 len 40960 data: 3 3 3 3 3 2 2 2 2 2
12002024-02-07T18:54:56.842Z16/30 Write block 49 len 4096 data: 3
12012024-02-07T18:54:56.842Z17/30 Read block 547 len 40960
12022024-02-07T18:54:56.842Z18/30 Flush
12032024-02-07T18:54:56.842Z19/30 Write block 424 len 8192 data: 3 3
12042024-02-07T18:54:56.842Z20/30 Read block 232 len 24576
12052024-02-07T18:54:56.842Z21/30 Write block 490 len 36864 data: 3 3 3 3 3 2 2 2 2
12062024-02-07T18:54:56.842Z22/30 Write block 286 len 20480 data: 3 2 2 2 2
12072024-02-07T18:54:56.842Z23/30 Read block 68 len 28672
12082024-02-07T18:54:56.842Z24/30 Write block 13 len 28672 data: 2 2 2 2 2 2 2
12092024-02-07T18:54:57.144Z25/30 Write block 191 len 32768 data: 4 4 3 3 3 4 4 4
12102024-02-07T18:54:57.144Z26/30 Write block 437 len 16384 data: 3 2 2 2
12112024-02-07T18:54:57.144Z27/30 Write block 88 len 28672 data: 2 2 2 2 2 2 3
12122024-02-07T18:54:57.144Z28/30 Read block 367 len 32768
12132024-02-07T18:54:57.144Z29/30 Read block 325 len 28672
12142024-02-07T18:54:57.144Z30/30 Write block 235 len 40960 data: 3 3 3 3 3 2 2 2 2 2
12152024-02-07T18:54:57.144Z----------------------------------------------------------------
12162024-02-07T18:54:57.144Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
12172024-02-07T18:54:57.144ZStates: Active Active Active
12182024-02-07T18:54:57.144ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
12192024-02-07T18:54:57.145Z 1 Acked 1000 Write 5 Done Done Sent false
12202024-02-07T18:54:57.145Z 2 Acked 1001 Write 6 Done Done Sent false
12212024-02-07T18:54:57.145Z 3 Acked 1002 Read 2 Done Done Sent false
12222024-02-07T18:54:57.145Z 4 Acked 1003 Read 9 Done Done Sent false
12232024-02-07T18:54:57.145Z 5 Acked 1004 Read 5 Done Done Sent false
12242024-02-07T18:54:57.145Z 6 Acked 1005 Read 1 Done Done Sent false
12252024-02-07T18:54:57.145Z 7 Acked 1006 Write 2 Done Done Sent false
12262024-02-07T18:54:57.145Z 8 Acked 1007 Read 9 Done Done Sent false
12272024-02-07T18:54:57.145Z 9 Acked 1008 Write 4 Done Done Sent false
12282024-02-07T18:54:57.145Z 10 Acked 1009 Read 2 Done Done Sent false
12292024-02-07T18:54:57.145Z 11 Acked 1010 Read 5 Done Done Sent false
12302024-02-07T18:54:57.145Z 12 Acked 1011 Read 5 Done Done Sent false
12312024-02-07T18:54:57.145Z 13 Acked 1012 Read 9 Done Done Sent false
12322024-02-07T18:54:57.145Z 14 Acked 1013 Read 7 Done Done Sent false
12332024-02-07T18:54:57.145Z 15 Acked 1014 Write 10 Done Done Sent false
12342024-02-07T18:54:57.145Z 16 Acked 1015 Write 1 Done Done Sent false
12352024-02-07T18:54:57.145Z 17 Acked 1016 Read 10 Done Done Sent false
12362024-02-07T18:54:57.145Z 18 Acked 1017 Flush 0 Done Done Sent false
12372024-02-07T18:54:57.145Z 19 Acked 1018 Write 2 Done Done Sent false
12382024-02-07T18:54:57.145Z 20 Acked 1019 Read 6 Done Done Sent false
12392024-02-07T18:54:57.145Z 21 Acked 1020 Write 9 Done Done Sent false
12402024-02-07T18:54:57.146Z 22 Acked 1021 Write 5 Done Done Sent false
12412024-02-07T18:54:57.146Z 23 Acked 1022 Read 7 Done Done Sent false
12422024-02-07T18:54:57.146Z 24 Acked 1023 Write 7 Done Done Sent false
12432024-02-07T18:54:57.146Z 25 Acked 1024 Write 8 Done Done Sent false
12442024-02-07T18:54:57.146Z 26 Acked 1025 Write 4 Done Done Sent false
12452024-02-07T18:54:57.146Z 27 Acked 1026 Write 7 Done Done Sent false
12462024-02-07T18:54:57.146Z 28 Acked 1027 Read 8 Done Done Sent false
12472024-02-07T18:54:57.146Z 29 Acked 1028 Read 7 Done Done Sent false
12482024-02-07T18:54:57.146Z 30 Acked 1029 Write 10 Sent Sent Sent false
12492024-02-07T18:54:57.146Z STATES DS:0 DS:1 DS:2 TOTAL
12502024-02-07T18:54:57.146Z New 0 0 0 0
12512024-02-07T18:54:57.146Z Sent 1 1 30 32
12522024-02-07T18:54:57.146Z Done 29 29 0 58
12532024-02-07T18:54:57.146Z Skipped 0 0 0 0
12542024-02-07T18:54:57.146Z Error 0 0 0 0
12552024-02-07T18:54:57.146ZLast Flush: 1017 1017 0
12562024-02-07T18:54:57.147ZDownstairs last five completed:
12572024-02-07T18:54:57.147ZUpstairs last five completed: 30 29 28 27 26
12582024-02-07T18:54:57.147Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:53.435088338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1175}
12592024-02-07T18:54:57.147ZWrote out file "/tmp/test_repair_verify.data"
12602024-02-07T18:54:57.147Z
12612024-02-07T18:54:57.147ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12622024-02-07T18:54:57.147ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12632024-02-07T18:54:57.147Z 0 000-019 1 1 1 1 1 1 T T F <---
12642024-02-07T18:54:57.147Z 1 020-039 6 6 6 14 14 14 F F F
12652024-02-07T18:54:57.147Z 2 040-059 10 10 8 21 21 19 F F F <---
12662024-02-07T18:54:57.147Z 3 060-079 7 7 7 17 17 17 F F F
12672024-02-07T18:54:57.147Z 4 080-099 6 6 6 16 16 16 T T F <---
12682024-02-07T18:54:57.147Z 5 100-119 9 9 9 20 20 20 F F F
12692024-02-07T18:54:57.147Z 6 120-139 5 5 5 13 13 13 F F F
12702024-02-07T18:54:57.147Z 7 140-159 6 6 6 14 14 14 F F F
12712024-02-07T18:54:57.147Z 8 160-179 9 9 9 20 20 20 F F F
12722024-02-07T18:54:57.147Z 9 180-199 9 9 9 20 20 20 T T F <---
12732024-02-07T18:54:57.147Z 10 200-219 8 8 8 19 19 19 F F F
12742024-02-07T18:54:57.147Z 11 220-239 9 9 9 20 20 20 T T F <---
12752024-02-07T18:54:57.147Z 12 240-259 10 10 1 21 21 1 T T F <---
12762024-02-07T18:54:57.147Z 13 260-279 6 6 6 16 16 16 F F F
12772024-02-07T18:54:57.147Z 14 280-299 6 6 6 16 16 16 T T F <---
12782024-02-07T18:54:57.148Z 15 300-319 10 10 5 21 21 13 F F F <---
12792024-02-07T18:54:57.148Z 16 320-339 10 10 6 21 21 15 F F F <---
12802024-02-07T18:54:57.148Z 17 340-359 10 10 9 21 21 20 F F F <---
12812024-02-07T18:54:57.148Z 18 360-379 10 10 4 21 21 12 F F F <---
12822024-02-07T18:54:57.148Z 19 380-399 9 9 9 20 20 20 F F F
12832024-02-07T18:54:57.148Z 20 400-419 9 9 9 20 20 20 F F F
12842024-02-07T18:54:57.148Z 21 420-439 9 9 9 20 20 20 T T F <---
12852024-02-07T18:54:57.148Z 22 440-459 9 9 9 20 20 20 T T F <---
12862024-02-07T18:54:57.148Z 23 460-479 9 9 9 20 20 20 F F F
12872024-02-07T18:54:57.148Z 24 480-499 4 4 4 10 10 10 T T F <---
12882024-02-07T18:54:57.148Z 25 500-519 4 4 4 10 10 10 F F F
12892024-02-07T18:54:57.148Z 26 520-539 1 1 1 1 1 1 F F F
12902024-02-07T18:54:57.148Z 27 540-559 10 10 9 21 21 20 F F F <---
12912024-02-07T18:54:57.148Z 28 560-579 10 10 9 21 21 20 F F F <---
12922024-02-07T18:54:57.148Z 29 580-599 4 4 4 9 9 9 F F F
12932024-02-07T18:54:57.148ZMax gen: 10, Max flush: 21
12942024-02-07T18:54:57.148ZA Difference in extent metadata is expected here
12952024-02-07T18:54:57.148ZOn loop 5
12962024-02-07T18:54:57.148Z
12972024-02-07T18:54:57.148ZVerifying data now
12982024-02-07T18:54:59.046ZNo change after verify
12992024-02-07T18:54:59.046ZLoop: 5 Downstairs dump after verify (and repair):
13002024-02-07T18:54:59.084ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
13012024-02-07T18:54:59.084Z 0 000-019 11 11 11 22 22 22 F F F
13022024-02-07T18:54:59.084Z 1 020-039 6 6 6 14 14 14 F F F
13032024-02-07T18:54:59.084Z 2 040-059 10 10 10 21 21 21 F F F
13042024-02-07T18:54:59.084Z 3 060-079 7 7 7 17 17 17 F F F
13052024-02-07T18:54:59.084Z 4 080-099 11 11 11 22 22 22 F F F
13062024-02-07T18:54:59.084Z 5 100-119 9 9 9 20 20 20 F F F
13072024-02-07T18:54:59.084Z 6 120-139 5 5 5 13 13 13 F F F
13082024-02-07T18:54:59.084Z 7 140-159 6 6 6 14 14 14 F F F
13092024-02-07T18:54:59.084Z 8 160-179 9 9 9 20 20 20 F F F
13102024-02-07T18:54:59.085Z 9 180-199 11 11 11 22 22 22 F F F
13112024-02-07T18:54:59.085Z 10 200-219 8 8 8 19 19 19 F F F
13122024-02-07T18:54:59.085Z 11 220-239 11 11 11 22 22 22 F F F
13132024-02-07T18:54:59.085Z 12 240-259 11 11 11 22 22 22 F F F
13142024-02-07T18:54:59.085Z 13 260-279 6 6 6 16 16 16 F F F
13152024-02-07T18:54:59.085Z 14 280-299 11 11 11 22 22 22 F F F
13162024-02-07T18:54:59.085Z 15 300-319 10 10 10 21 21 21 F F F
13172024-02-07T18:54:59.085Z 16 320-339 10 10 10 21 21 21 F F F
13182024-02-07T18:54:59.085Z 17 340-359 10 10 10 21 21 21 F F F
13192024-02-07T18:54:59.085Z 18 360-379 10 10 10 21 21 21 F F F
13202024-02-07T18:54:59.085Z 19 380-399 9 9 9 20 20 20 F F F
13212024-02-07T18:54:59.085Z 20 400-419 9 9 9 20 20 20 F F F
13222024-02-07T18:54:59.086Z 21 420-439 11 11 11 22 22 22 F F F
13232024-02-07T18:54:59.086Z 22 440-459 11 11 11 22 22 22 F F F
13242024-02-07T18:54:59.086Z 23 460-479 9 9 9 20 20 20 F F F
13252024-02-07T18:54:59.086Z 24 480-499 11 11 11 22 22 22 F F F
13262024-02-07T18:54:59.086Z 25 500-519 4 4 4 10 10 10 F F F
13272024-02-07T18:54:59.086Z 26 520-539 1 1 1 1 1 1 F F F
13282024-02-07T18:54:59.086Z 27 540-559 10 10 10 21 21 21 F F F
13292024-02-07T18:54:59.086Z 28 560-579 10 10 10 21 21 21 F F F
13302024-02-07T18:54:59.086Z 29 580-599 4 4 4 9 9 9 F F F
13312024-02-07T18:54:59.086ZMax gen: 11, Max flush: 22
13322024-02-07T18:54:59.086Z
13332024-02-07T18:54:59.086ZBegin loop 6
13342024-02-07T18:54:59.086ZDownstairs to restart: 2
13352024-02-07T18:54:59.086Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.262886336Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183}
13362024-02-07T18:54:59.086Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263124004Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183}
13372024-02-07T18:54:59.086Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.26315971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183}
13382024-02-07T18:54:59.086Z{"msg":"Crucible 75097c5c-36b1-48aa-8dba-d43b8de97c7d has session id: da8f006b-09ec-4261-aa27-f34ec6f350c6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263175737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13392024-02-07T18:54:59.086Z{"msg":"Upstairs opts: Upstairs UUID: 75097c5c-36b1-48aa-8dba-d43b8de97c7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263205284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13402024-02-07T18:54:59.086Z{"msg":"Crucible stats registered with UUID: 75097c5c-36b1-48aa-8dba-d43b8de97c7d","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263234369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13412024-02-07T18:54:59.086ZCrucible runtime is spawned
13422024-02-07T18:54:59.087Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263366182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183}
13432024-02-07T18:54:59.087Z{"msg":"75097c5c-36b1-48aa-8dba-d43b8de97c7d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263516371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13442024-02-07T18:54:59.087Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263572624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13452024-02-07T18:54:59.087Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263612338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13462024-02-07T18:54:59.087Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263647104Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13472024-02-07T18:54:59.087Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263695707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13482024-02-07T18:54:59.087Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263752539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13492024-02-07T18:54:59.087Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263791254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13502024-02-07T18:54:59.087Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.26382492Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13512024-02-07T18:54:59.087Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263855716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13522024-02-07T18:54:59.087Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263903909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13532024-02-07T18:54:59.087Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.263936634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13542024-02-07T18:54:59.087Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.26396835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13552024-02-07T18:54:59.087Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.264008755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13562024-02-07T18:54:59.087Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.264704461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13572024-02-07T18:54:59.088Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.264779031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13582024-02-07T18:54:59.088Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.264870178Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13592024-02-07T18:54:59.088Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.264913522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13602024-02-07T18:54:59.088Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265271384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13612024-02-07T18:54:59.088Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265406616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13622024-02-07T18:54:59.088Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265880332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13632024-02-07T18:54:59.088Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265921056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13642024-02-07T18:54:59.088Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265936824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13652024-02-07T18:54:59.088Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265957851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13662024-02-07T18:54:59.088Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265972429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13672024-02-07T18:54:59.088Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.265986787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13682024-02-07T18:54:59.088Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.266007454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13692024-02-07T18:54:59.088Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.266021023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13702024-02-07T18:54:59.088Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.266591606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13712024-02-07T18:54:59.088Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267093718Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13722024-02-07T18:54:59.089Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267146091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13732024-02-07T18:54:59.089Z{"msg":"[0]R flush_numbers[0..12]: [22, 14, 21, 17, 22, 20, 13, 14, 20, 22, 19, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267180536Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13742024-02-07T18:54:59.089Z{"msg":"[0]R generation[0..12]: [11, 6, 10, 7, 11, 9, 5, 6, 9, 11, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267195814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13752024-02-07T18:54:59.089Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267218031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13762024-02-07T18:54:59.089Z{"msg":"[1]R flush_numbers[0..12]: [22, 14, 21, 17, 22, 20, 13, 14, 20, 22, 19, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267244787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13772024-02-07T18:54:59.089Z{"msg":"[1]R generation[0..12]: [11, 6, 10, 7, 11, 9, 5, 6, 9, 11, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267262665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"Wait for a query_work_queue command to finish before sending IO
13782024-02-07T18:54:59.089Zdownstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13792024-02-07T18:54:59.089Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267293851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13802024-02-07T18:54:59.089Z{"msg":"[2]R flush_numbers[0..12]: [22, 14, 21, 17, 22, 20, 13, 14, 20, 22, 19, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267319397Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13812024-02-07T18:54:59.089Z{"msg":"[2]R generation[0..12]: [11, 6, 10, 7, 11, 9, 5, 6, 9, 11, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267337565Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13822024-02-07T18:54:59.089Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267355462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13832024-02-07T18:54:59.089Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267380509Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13842024-02-07T18:54:59.089Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267397676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13852024-02-07T18:54:59.089Z{"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267422113Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13862024-02-07T18:54:59.089Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267439591Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13872024-02-07T18:54:59.089Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267480355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13882024-02-07T18:54:59.089Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267497053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13892024-02-07T18:54:59.089Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267514501Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13902024-02-07T18:54:59.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267539577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13912024-02-07T18:54:59.090Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267556635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13922024-02-07T18:54:59.090Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267580172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13932024-02-07T18:54:59.090Z{"msg":"75097c5c-36b1-48aa-8dba-d43b8de97c7d is now active with session: da8f006b-09ec-4261-aa27-f34ec6f350c6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267596759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13942024-02-07T18:54:59.090Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267620396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183,"session_id":"da8f006b-09ec-4261-aa27-f34ec6f350c6"}
13952024-02-07T18:54:59.090Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:54:58.267636214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183}
13962024-02-07T18:54:59.090ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13972024-02-07T18:54:59.090ZLoading write count information from file "/tmp/test_repair_verify.data"
13982024-02-07T18:54:59.090ZRun Repair workload
13992024-02-07T18:54:59.090Z01/30 Write block 139 len 4096 data: 3
14002024-02-07T18:54:59.172Z02/30 Write block 354 len 4096 data: 2
14012024-02-07T18:54:59.172Z03/30 Flush
14022024-02-07T18:54:59.172Z04/30 Write block 314 len 12288 data: 3 3 4
14032024-02-07T18:54:59.172Z05/30 Write block 172 len 36864 data: 4 4 3 3 3 3 2 3 3
14042024-02-07T18:54:59.172Z06/30 Write block 471 len 32768 data: 2 2 2 2 2 2 2 2
14052024-02-07T18:54:59.172Z07/30 Read block 418 len 16384
14062024-02-07T18:54:59.173Z08/30 Read block 581 len 4096
14072024-02-07T18:54:59.173Z09/30 Flush
14082024-02-07T18:54:59.173Z10/30 Write block 109 len 4096 data: 2
14092024-02-07T18:54:59.173Z11/30 Read block 469 len 12288
14102024-02-07T18:54:59.173Z12/30 Flush
14112024-02-07T18:54:59.173Z13/30 Write block 219 len 24576 data: 2 2 2 2 2 2
14122024-02-07T18:54:59.173Z14/30 Flush
14132024-02-07T18:54:59.173Z15/30 Write block 69 len 12288 data: 5 4 3
14142024-02-07T18:54:59.173Z16/30 Write block 108 len 4096 data: 2
14152024-02-07T18:54:59.173Z17/30 Write block 426 len 16384 data: 3 3 3 2
14162024-02-07T18:54:59.173Z18/30 Read block 127 len 8192
14172024-02-07T18:54:59.173Z19/30 Read block 164 len 32768
14182024-02-07T18:54:59.173Z20/30 Read block 375 len 32768
14192024-02-07T18:54:59.173Z21/30 Write block 512 len 36864 data: 2 2 2 2 2 2 2 2 2
14202024-02-07T18:54:59.173Z22/30 Flush
14212024-02-07T18:54:59.173Z23/30 Write block 83 len 4096 data: 3
14222024-02-07T18:54:59.173Z24/30 Read block 136 len 24576
14232024-02-07T18:54:59.174Z25/30 Read block 511 len 36864
14242024-02-07T18:54:59.174Z26/30 Write block 119 len 24576 data: 4 3 3 2 2 2
14252024-02-07T18:54:59.174Z27/30 Write block 409 len 28672 data: 2 2 3 2 2 2 2
14262024-02-07T18:54:59.174Z28/30 Read block 494 len 8192
14272024-02-07T18:54:59.174Z29/30 Read block 473 len 32768
14282024-02-07T18:54:59.174Z30/30 Read block 148 len 12288
14292024-02-07T18:54:59.174Z----------------------------------------------------------------
14302024-02-07T18:54:59.174Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14312024-02-07T18:54:59.174ZStates: Active Active Active
14322024-02-07T18:54:59.174ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14332024-02-07T18:54:59.174Z 1 Acked 1000 Write 1 Done Done Done false
14342024-02-07T18:54:59.174Z 2 Acked 1001 Write 1 Done Done Sent false
14352024-02-07T18:54:59.174Z 3 Acked 1002 Flush 0 Done Done Sent false
14362024-02-07T18:54:59.174Z 4 Acked 1003 Write 3 Done Done Sent false
14372024-02-07T18:54:59.174Z 5 Acked 1004 Write 9 Done Done Sent false
14382024-02-07T18:54:59.174Z 6 Acked 1005 Write 8 Done Done Sent false
14392024-02-07T18:54:59.174Z 7 Acked 1006 Read 4 Done Done Sent false
14402024-02-07T18:54:59.174Z 8 Acked 1007 Read 1 Done Done Sent false
14412024-02-07T18:54:59.174Z 9 Acked 1008 Flush 0 Done Done Sent false
14422024-02-07T18:54:59.174Z 10 Acked 1009 Write 1 Done Done Sent false
14432024-02-07T18:54:59.174Z 11 Acked 1010 Read 3 Done Done Sent false
14442024-02-07T18:54:59.174Z 12 Acked 1011 Flush 0 Done Done Sent false
14452024-02-07T18:54:59.174Z 13 Acked 1012 Write 6 Done Done Sent false
14462024-02-07T18:54:59.175Z 14 Acked 1013 Flush 0 Done Done Sent false
14472024-02-07T18:54:59.175Z 15 Acked 1014 Write 3 Done Done Sent false
14482024-02-07T18:54:59.175Z 16 Acked 1015 Write 1 Done Done Sent false
14492024-02-07T18:54:59.175Z 17 Acked 1016 Write 4 Done Done Sent false
14502024-02-07T18:54:59.175Z 18 Acked 1017 Read 2 Done Done Sent false
14512024-02-07T18:54:59.175Z 19 Acked 1018 Read 8 Done Done Sent false
14522024-02-07T18:54:59.175Z 20 Acked 1019 Read 8 Done Done Sent false
14532024-02-07T18:54:59.175Z 21 Acked 1020 Write 9 Done Done Sent false
14542024-02-07T18:54:59.175Z 22 Acked 1021 Flush 0 Done Done Sent false
14552024-02-07T18:54:59.175Z 23 Acked 1022 Write 1 Done Done Sent false
14562024-02-07T18:54:59.175Z 24 Acked 1023 Read 6 Done Done Sent false
14572024-02-07T18:54:59.175Z 25 Acked 1024 Read 9 Done Done Sent false
14582024-02-07T18:54:59.175Z 26 Acked 1025 Write 6 Done Done Sent false
14592024-02-07T18:54:59.175Z 27 Acked 1026 Write 7 Done Done Sent false
14602024-02-07T18:54:59.175Z 28 Acked 1027 Read 2 Done Done Sent false
14612024-02-07T18:54:59.175Z 29 Acked 1028 Read 8 Done Done Sent false
14622024-02-07T18:54:59.176Z 30 Acked 1029 Read 3 Done Done Sent false
14632024-02-07T18:54:59.176Z STATES DS:0 DS:1 DS:2 TOTAL
14642024-02-07T18:54:59.176Z New 0 0 0 0
14652024-02-07T18:54:59.176Z Sent 0 0 29 29
14662024-02-07T18:54:59.176Z Done 30 30 1 61
14672024-02-07T18:54:59.176Z Skipped 0 0 0 0
14682024-02-07T18:54:59.176Z Error 0 0 0 0
14692024-02-07T18:54:59.176ZLast Flush: 1021 1021 0
14702024-02-07T18:54:59.176ZDownstairs last five completed:
14712024-02-07T18:54:59.176ZUpstairs last five completed: 30 29 28 27 26
14722024-02-07T18:54:59.176Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:54:58.298995613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1183}
14732024-02-07T18:54:59.176ZWrote out file "/tmp/test_repair_verify.data"
14742024-02-07T18:54:59.176Z
14752024-02-07T18:54:59.176ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14762024-02-07T18:54:59.176ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14772024-02-07T18:54:59.176Z 0 000-019 11 11 11 22 22 22 F F F
14782024-02-07T18:54:59.176Z 1 020-039 6 6 6 14 14 14 F F F
14792024-02-07T18:54:59.176Z 2 040-059 10 10 10 21 21 21 F F F
14802024-02-07T18:54:59.176Z 3 060-079 12 12 7 27 27 17 F F F <---
14812024-02-07T18:54:59.177Z 4 080-099 11 11 11 22 22 22 T T F <---
14822024-02-07T18:54:59.177Z 5 100-119 12 12 9 27 27 20 T T F <---
14832024-02-07T18:54:59.177Z 6 120-139 12 12 5 23 23 13 T T T <---
14842024-02-07T18:54:59.177Z 7 140-159 6 6 6 14 14 14 F F F
14852024-02-07T18:54:59.177Z 8 160-179 12 12 9 24 24 20 F F F <---
14862024-02-07T18:54:59.177Z 9 180-199 12 12 11 24 24 22 F F F <---
14872024-02-07T18:54:59.177Z 10 200-219 12 12 8 26 26 19 F F F <---
14882024-02-07T18:54:59.177Z 11 220-239 12 12 11 26 26 22 F F F <---
14892024-02-07T18:54:59.177Z 12 240-259 11 11 11 22 22 22 F F F
14902024-02-07T18:54:59.177Z 13 260-279 6 6 6 16 16 16 F F F
14912024-02-07T18:54:59.177Z 14 280-299 11 11 11 22 22 22 F F F
14922024-02-07T18:54:59.177Z 15 300-319 12 12 10 24 24 21 F F F <---
14932024-02-07T18:54:59.177Z 16 320-339 10 10 10 21 21 21 F F F
14942024-02-07T18:54:59.177Z 17 340-359 12 12 10 23 23 21 F F F <---
14952024-02-07T18:54:59.177Z 18 360-379 10 10 10 21 21 21 F F F
14962024-02-07T18:54:59.177Z 19 380-399 9 9 9 20 20 20 F F F
14972024-02-07T18:54:59.177Z 20 400-419 9 9 9 20 20 20 T T F <---
14982024-02-07T18:54:59.178Z 21 420-439 12 12 11 27 27 22 F F F <---
14992024-02-07T18:54:59.178Z 22 440-459 11 11 11 22 22 22 F F F
15002024-02-07T18:54:59.216Z 23 460-479 12 12 9 24 24 20 F F F <---
15012024-02-07T18:54:59.216Z 24 480-499 11 11 11 22 22 22 F F F
15022024-02-07T18:54:59.216Z 25 500-519 12 12 4 27 27 10 F F F <---
15032024-02-07T18:54:59.216Z 26 520-539 12 12 1 27 27 1 F F F <---
15042024-02-07T18:54:59.216Z 27 540-559 10 10 10 21 21 21 F F F
15052024-02-07T18:54:59.216Z 28 560-579 10 10 10 21 21 21 F F F
15062024-02-07T18:54:59.216Z 29 580-599 4 4 4 9 9 9 F F F
15072024-02-07T18:54:59.216ZMax gen: 12, Max flush: 27
15082024-02-07T18:54:59.217ZError: Difference in extent metadata found!
15092024-02-07T18:54:59.217ZA Difference in extent metadata is expected here
15102024-02-07T18:54:59.217ZOn loop 6
15112024-02-07T18:54:59.217Z
15122024-02-07T18:54:59.217ZVerifying data now
15132024-02-07T18:55:13.578ZNo change after verify
15142024-02-07T18:55:13.578ZLoop: 6 Downstairs dump after verify (and repair):
15152024-02-07T18:55:13.701ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
15162024-02-07T18:55:13.701Z 0 000-019 11 11 11 22 22 22 F F F
15172024-02-07T18:55:13.701Z 1 020-039 6 6 6 14 14 14 F F F
15182024-02-07T18:55:13.701Z 2 040-059 10 10 10 21 21 21 F F F
15192024-02-07T18:55:13.702Z 3 060-079 12 12 12 27 27 27 F F F
15202024-02-07T18:55:13.702Z 4 080-099 13 13 13 28 28 28 F F F
15212024-02-07T18:55:13.702Z 5 100-119 13 13 13 28 28 28 F F F
15222024-02-07T18:55:13.702Z 6 120-139 13 13 13 28 28 28 F F F
15232024-02-07T18:55:13.702Z 7 140-159 6 6 6 14 14 14 F F F
15242024-02-07T18:55:13.702Z 8 160-179 12 12 12 24 24 24 F F F
15252024-02-07T18:55:13.702Z 9 180-199 12 12 12 24 24 24 F F F
15262024-02-07T18:55:13.702Z 10 200-219 12 12 12 26 26 26 F F F
15272024-02-07T18:55:13.702Z 11 220-239 12 12 12 26 26 26 F F F
15282024-02-07T18:55:13.702Z 12 240-259 11 11 11 22 22 22 F F F
15292024-02-07T18:55:13.702Z 13 260-279 6 6 6 16 16 16 F F F
15302024-02-07T18:55:13.702Z 14 280-299 11 11 11 22 22 22 F F F
15312024-02-07T18:55:13.703Z 15 300-319 12 12 12 24 24 24 F F F
15322024-02-07T18:55:13.703Z 16 320-339 10 10 10 21 21 21 F F F
15332024-02-07T18:55:13.703Z 17 340-359 12 12 12 23 23 23 F F F
15342024-02-07T18:55:13.703Z 18 360-379 10 10 10 21 21 21 F F F
15352024-02-07T18:55:13.703Z 19 380-399 9 9 9 20 20 20 F F F
15362024-02-07T18:55:13.703Z 20 400-419 13 13 13 28 28 28 F F F
15372024-02-07T18:55:13.703Z 21 420-439 12 12 12 27 27 27 F F F
15382024-02-07T18:55:13.703Z 22 440-459 11 11 11 22 22 22 F F F
15392024-02-07T18:55:13.703Z 23 460-479 12 12 12 24 24 24 F F F
15402024-02-07T18:55:13.703Z 24 480-499 11 11 11 22 22 22 F F F
15412024-02-07T18:55:13.703Z 25 500-519 12 12 12 27 27 27 F F F
15422024-02-07T18:55:13.703Z 26 520-539 12 12 12 27 27 27 F F F
15432024-02-07T18:55:13.704Z 27 540-559 10 10 10 21 21 21 F F F
15442024-02-07T18:55:13.704Z 28 560-579 10 10 10 21 21 21 F F F
15452024-02-07T18:55:13.704Z 29 580-599 4 4 4 9 9 9 F F F
15462024-02-07T18:55:13.704ZMax gen: 13, Max flush: 28
15472024-02-07T18:55:13.704Z
15482024-02-07T18:55:13.704ZBegin loop 7
15492024-02-07T18:55:13.704ZDownstairs to restart: 0
15502024-02-07T18:55:13.704Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.06940523Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191}
15512024-02-07T18:55:13.704Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.069669154Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191}
15522024-02-07T18:55:13.705Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
15532024-02-07T18:55:13.705Z,"time":"2024-02-07T18:55:04.069712138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191}
15542024-02-07T18:55:13.705Z{"msg":"Crucible c0bdc9da-39b7-4252-81ce-ed9117c2232f has session id: a4176187-b34e-4484-bd45-9f3edee7426b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.069753343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15552024-02-07T18:55:13.705Z{"msg":"Upstairs opts: Upstairs UUID: c0bdc9da-39b7-4252-81ce-ed9117c2232f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.069792558Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15562024-02-07T18:55:13.705Z{"msg":"Crucible stats registered with UUID: c0bdc9da-39b7-4252-81ce-ed9117c2232f","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.069833532Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15572024-02-07T18:55:13.705Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.069861928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191}
15582024-02-07T18:55:13.705Z{"msg":"c0bdc9da-39b7-4252-81ce-ed9117c2232f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.069958185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15592024-02-07T18:55:13.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.06999729Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15602024-02-07T18:55:13.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070056222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15612024-02-07T18:55:13.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070106505Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15622024-02-07T18:55:13.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070196273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15632024-02-07T18:55:13.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070240437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15642024-02-07T18:55:13.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070379778Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15652024-02-07T18:55:13.706Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070411544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15662024-02-07T18:55:13.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070427642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15672024-02-07T18:55:13.706Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070449539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15682024-02-07T18:55:13.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070484594Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15692024-02-07T18:55:13.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070532288Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15702024-02-07T18:55:13.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070577792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15712024-02-07T18:55:13.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070938503Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15722024-02-07T18:55:13.706Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.070980657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15732024-02-07T18:55:13.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.071015642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15742024-02-07T18:55:13.706Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.071051867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15752024-02-07T18:55:13.707Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.071575706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15762024-02-07T18:55:13.707Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.071727406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15772024-02-07T18:55:13.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072108094Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15782024-02-07T18:55:13.707Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072137661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15792024-02-07T18:55:13.707Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072153618Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15802024-02-07T18:55:13.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072193903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15812024-02-07T18:55:13.707Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072226398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15822024-02-07T18:55:13.707Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072258294Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15832024-02-07T18:55:13.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072297759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15842024-02-07T18:55:13.707Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072330624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15852024-02-07T18:55:13.707Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.072819248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15862024-02-07T18:55:13.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073285795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15872024-02-07T18:55:13.708Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.07332281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15882024-02-07T18:55:13.708Z{"msg":"[0]R flush_numbers[0..12]: [22, 14, 21, 27, 28, 28, 28, 14, 24, 24, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073340858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15892024-02-07T18:55:13.708Z{"msg":"[0]R generation[0..12]: [11, 6, 10, 12, 13, 13, 13, 6, 12, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073362735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15902024-02-07T18:55:13.708Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073384572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15912024-02-07T18:55:13.708Z{"msg":"[1]R flush_numbers[0..12]: [22, 14, 21, 27, 28, 28, 28, 14, 24, 24, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073405959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15922024-02-07T18:55:13.708Z{"msg":"[1]R generation[0..12]: [11, 6, 10, 12, 13, 13, 13, 6, 12, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073427016Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15932024-02-07T18:55:13.708Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15942024-02-07T18:55:13.708Z,"time":"2024-02-07T18:55:04.073448533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15952024-02-07T18:55:13.708Z{"msg":"[2]R flush_numbers[0..12]: [22, 14, 21, 27, 28, 28, 28, 14, 24, 24, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073465901Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15962024-02-07T18:55:13.708Z{"msg":"[2]R generation[0..12]: [11, 6, 10, 12, 13, 13, 13, 6, 12, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073486858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15972024-02-07T18:55:13.709Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073511255Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15982024-02-07T18:55:13.709Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073541611Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
15992024-02-07T18:55:13.709Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073563848Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16002024-02-07T18:55:13.709Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073577376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16012024-02-07T18:55:13.709Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073602862Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16022024-02-07T18:55:13.709Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073629209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16032024-02-07T18:55:13.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073649226Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16042024-02-07T18:55:13.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073670533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16052024-02-07T18:55:13.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.07369094Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16062024-02-07T18:55:13.709Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073711148Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16072024-02-07T18:55:13.709Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073730475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16082024-02-07T18:55:13.709Z{"msg":"c0bdc9da-39b7-4252-81ce-ed9117c2232f is now active with session: a4176187-b34e-4484-bd45-9f3edee7426b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073750542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16092024-02-07T18:55:13.710Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.07376927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191,"session_id":"a4176187-b34e-4484-bd45-9f3edee7426b"}
16102024-02-07T18:55:13.710Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:04.073788667Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191}
16112024-02-07T18:55:13.710ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
16122024-02-07T18:55:13.710ZLoading write count information from file "/tmp/test_repair_verify.data"
16132024-02-07T18:55:13.710ZError: Difference in extent metadata found!
16142024-02-07T18:55:13.710ZRun Repair workload
16152024-02-07T18:55:13.782Z01/30 Write block 493 len 32768 data: 4 4 3 3 3 3 2 2
16162024-02-07T18:55:13.782Z02/30 Flush
16172024-02-07T18:55:13.782Z03/30 Write block 71 len 20480 data: 4 2 2 2 2
16182024-02-07T18:55:13.782Z04/30 Read block 347 len 8192
16192024-02-07T18:55:13.782Z05/30 Write block 428 len 16384 data: 4 3 2 2
16202024-02-07T18:55:13.782Z06/30 Write block 331 len 12288 data: 2 2 2
16212024-02-07T18:55:13.783Z07/30 Read block 406 len 4096
16222024-02-07T18:55:13.783Z08/30 Read block 135 len 32768
16232024-02-07T18:55:13.783Z09/30 Write block 19 len 32768 data: 3 2 2 2 2 2 2 2
16242024-02-07T18:55:13.783Z10/30 Read block 83 len 36864
16252024-02-07T18:55:13.783Z11/30 Write block 243 len 40960 data: 3 3 2 2 3 3 2 2 2 2
16262024-02-07T18:55:13.783Z12/30 Read block 114 len 16384
16272024-02-07T18:55:13.783Z13/30 Write block 117 len 8192 data: 2 3
16282024-02-07T18:55:13.783Z14/30 Write block 417 len 24576 data: 3 3 3 3 3 3
16292024-02-07T18:55:13.783Z15/30 Flush
16302024-02-07T18:55:13.783Z16/30 Write block 234 len 32768 data: 3 4 4 4 4 4 3 3
16312024-02-07T18:55:13.783Z17/30 Write block 213 len 8192 data: 2 2
16322024-02-07T18:55:13.783Z18/30 Read block 511 len 20480
16332024-02-07T18:55:13.783Z19/30 Read block 44 len 4096
16342024-02-07T18:55:13.783Z20/30 Read block 379 len 40960
16352024-02-07T18:55:13.783Z21/30 Write block 230 len 24576 data: 2 2 3 3 4 5
16362024-02-07T18:55:13.783Z22/30 Write block 287 len 16384 data: 3 3 3 3
16372024-02-07T18:55:13.783Z23/30 Read block 117 len 40960
16382024-02-07T18:55:13.783Z24/30 Write block 164 len 24576 data: 5 5 5 5 5 5
16392024-02-07T18:55:13.783Z25/30 Read block 429 len 12288
16402024-02-07T18:55:13.783Z26/30 Write block 78 len 4096 data: 4
16412024-02-07T18:55:13.783Z27/30 Write block 106 len 8192 data: 2 2
16422024-02-07T18:55:13.783Z28/30 Read block 58 len 40960
16432024-02-07T18:55:13.784Z29/30 Read block 332 len 32768
16442024-02-07T18:55:13.784Z30/30 Write block 366 len 16384 data: 3 2 2 2
16452024-02-07T18:55:13.784Z----------------------------------------------------------------
16462024-02-07T18:55:13.784Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:28
16472024-02-07T18:55:13.784ZStates: Active Active Active
16482024-02-07T18:55:13.784ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16492024-02-07T18:55:13.784Z 3 Acked 1002 Write 5 Done Done Done false
16502024-02-07T18:55:13.784Z 4 Acked 1003 Read 2 Done Done Done false
16512024-02-07T18:55:13.784Z 5 Acked 1004 Write 4 Done Done Done false
16522024-02-07T18:55:13.784Z 6 Acked 1005 Write 3 Done Done Done false
16532024-02-07T18:55:13.784Z 7 Acked 1006 Read 1 Done Done Done false
16542024-02-07T18:55:13.784Z 8 Acked 1007 Read 8 Done Done Done false
16552024-02-07T18:55:13.784Z 9 Acked 1008 Write 8 Done Done Done false
16562024-02-07T18:55:13.784Z 10 Acked 1009 Read 9 Done Done Done false
16572024-02-07T18:55:13.784Z 11 Acked 1010 Write 10 Sent Done Done false
16582024-02-07T18:55:13.784Z 12 Acked 1011 Read 4 Sent Done Done false
16592024-02-07T18:55:13.784Z 13 Acked 1012 Write 2 Sent Done Done false
16602024-02-07T18:55:13.784Z 14 Acked 1013 Write 6 Sent Done Done false
16612024-02-07T18:55:13.784Z 15 Acked 1014 Flush 0 Sent Done Done false
16622024-02-07T18:55:13.784Z 16 Acked 1015 Write 8 Sent Done Done false
16632024-02-07T18:55:13.784Z 17 Acked 1016 Write 2 Sent Done Done false
16642024-02-07T18:55:13.784Z 18 Acked 1017 Read 5 Sent Done Done false
16652024-02-07T18:55:13.784Z 19 Acked 1018 Read 1 Sent Done Done false
16662024-02-07T18:55:13.784Z 20 Acked 1019 Read 10 Sent Done Done false
16672024-02-07T18:55:13.784Z 21 Acked 1020 Write 6 Sent Done Done false
16682024-02-07T18:55:13.785Z 22 Acked 1021 Write 4 Sent Done Done false
16692024-02-07T18:55:13.785Z 23 Acked 1022 Read 10 Sent Done Done false
16702024-02-07T18:55:13.785Z 24 Acked 1023 Write 6 Sent Done Done false
16712024-02-07T18:55:13.785Z 25 Acked 1024 Read 3 Sent Done Done false
16722024-02-07T18:55:13.785Z 26 Acked 1025 Write 1 Sent Done Done false
16732024-02-07T18:55:13.785Z 27 Acked 1026 Write 2 Sent Done Done false
16742024-02-07T18:55:13.785Z 28 Acked 1027 Read 10 Sent Done Done false
16752024-02-07T18:55:13.785Z 29 Acked 1028 Read 8 Sent Done Done false
16762024-02-07T18:55:13.785Z 30 Acked 1029 Write 4 Sent Sent Sent false
16772024-02-07T18:55:13.785Z STATES DS:0 DS:1 DS:2 TOTAL
16782024-02-07T18:55:13.785Z New 0 0 0 0
16792024-02-07T18:55:13.785Z Sent 20 1 1 22
16802024-02-07T18:55:13.785Z Done 8 27 27 62
16812024-02-07T18:55:13.785Z Skipped 0 0 0 0
16822024-02-07T18:55:13.785Z Error 0 0 0 0
16832024-02-07T18:55:13.785ZLast Flush: 1001 1014 1014
16842024-02-07T18:55:13.785ZDownstairs last five completed: 1001 1000
16852024-02-07T18:55:13.785ZUpstairs last five completed: 30 29 28 27 26
16862024-02-07T18:55:13.785Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:04.104106207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1191}
16872024-02-07T18:55:13.785ZWrote out file "/tmp/test_repair_verify.data"
16882024-02-07T18:55:13.785Z
16892024-02-07T18:55:13.785ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16902024-02-07T18:55:13.786ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16912024-02-07T18:55:13.786Z 0 000-019 11 14 14 22 30 30 T F F <---
16922024-02-07T18:55:13.786Z 1 020-039 6 14 14 14 30 30 T F F <---
16932024-02-07T18:55:13.786Z 2 040-059 10 10 10 21 21 21 F F F
16942024-02-07T18:55:13.786Z 3 060-079 12 14 14 27 30 30 T T T <---
16952024-02-07T18:55:13.786Z 4 080-099 13 13 13 28 28 28 F F F
16962024-02-07T18:55:13.786Z 5 100-119 13 14 14 28 30 30 F T T <---
16972024-02-07T18:55:13.786Z 6 120-139 13 13 13 28 28 28 F F F
16982024-02-07T18:55:13.786Z 7 140-159 6 6 6 14 14 14 F F F
16992024-02-07T18:55:13.786Z 8 160-179 12 12 12 24 24 24 F T T <---
17002024-02-07T18:55:13.786Z 9 180-199 12 12 12 24 24 24 F F F
17012024-02-07T18:55:13.786Z 10 200-219 12 12 12 26 26 26 F T T <---
17022024-02-07T18:55:13.786Z 11 220-239 12 12 12 26 26 26 F T T <---
17032024-02-07T18:55:13.786Z 12 240-259 11 14 14 22 30 30 F T T <---
17042024-02-07T18:55:13.786Z 13 260-279 6 6 6 16 16 16 F F F
17052024-02-07T18:55:13.786Z 14 280-299 11 11 11 22 22 22 F T T <---
17062024-02-07T18:55:13.786Z 15 300-319 12 12 12 24 24 24 F F F
17072024-02-07T18:55:13.786Z 16 320-339 10 14 14 21 30 30 T F F <---
17082024-02-07T18:55:13.786Z 17 340-359 12 12 12 23 23 23 F F F
17092024-02-07T18:55:13.786Z 18 360-379 10 10 10 21 21 21 F T T <---
17102024-02-07T18:55:13.786Z 19 380-399 9 9 9 20 20 20 F F F
17112024-02-07T18:55:13.786Z 20 400-419 13 14 14 28 30 30 F F F <---
17122024-02-07T18:55:13.786Z 21 420-439 12 14 14 27 30 30 T F F <---
17132024-02-07T18:55:13.786Z 22 440-459 11 11 11 22 22 22 F F F
17142024-02-07T18:55:13.786Z 23 460-479 12 12 12 24 24 24 F F F
17152024-02-07T18:55:13.804Z 24 480-499 14 14 14 29 29 29 F F F
17162024-02-07T18:55:13.804Z 25 500-519 14 14 14 29 29 29 F F F
17172024-02-07T18:55:13.804Z 26 520-539 12 12 12 27 27 27 F F F
17182024-02-07T18:55:13.804Z 27 540-559 10 10 10 21 21 21 F F F
17192024-02-07T18:55:13.804Z 28 560-579 10 10 10 21 21 21 F F F
17202024-02-07T18:55:13.804Z 29 580-599 4 4 4 9 9 9 F F F
17212024-02-07T18:55:13.804ZMax gen: 14, Max flush: 30
17222024-02-07T18:55:13.804ZA Difference in extent metadata is expected here
17232024-02-07T18:55:13.804ZOn loop 7
17242024-02-07T18:55:13.804Z
17252024-02-07T18:55:13.804ZVerifying data now
17262024-02-07T18:55:13.804ZNo change after verify
17272024-02-07T18:55:13.804ZLoop: 7 Downstairs dump after verify (and repair):
17282024-02-07T18:55:13.804ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17292024-02-07T18:55:13.804Z 0 000-019 14 14 14 30 30 30 F F F
17302024-02-07T18:55:13.804Z 1 020-039 14 14 14 30 30 30 F F F
17312024-02-07T18:55:13.804Z 2 040-059 10 10 10 21 21 21 F F F
17322024-02-07T18:55:13.805Z 3 060-079 15 15 15 31 31 31 F F F
17332024-02-07T18:55:13.805Z 4 080-099 13 13 13 28 28 28 F F F
17342024-02-07T18:55:13.805Z 5 100-119 15 15 15 31 31 31 F F F
17352024-02-07T18:55:13.805Z 6 120-139 13 13 13 28 28 28 F F F
17362024-02-07T18:55:13.805Z 7 140-159 6 6 6 14 14 14 F F F
17372024-02-07T18:55:13.805Z 8 160-179 15 15 15 31 31 31 F F F
17382024-02-07T18:55:13.805Z 9 180-199 12 12 12 24 24 24 F F F
17392024-02-07T18:55:13.805Z 10 200-219 15 15 15 31 31 31 F F F
17402024-02-07T18:55:13.805Z 11 220-239 15 15 15 31 31 31 F F F
17412024-02-07T18:55:13.805Z 12 240-259 15 15 15 31 31 31 F F F
17422024-02-07T18:55:13.805Z 13 260-279 6 6 6 16 16 16 F F F
17432024-02-07T18:55:13.805Z 14 280-299 15 15 15 31 31 31 F F F
17442024-02-07T18:55:13.805Z 15 300-319 12 12 12 24 24 24 F F F
17452024-02-07T18:55:13.805Z 16 320-339 14 14 14 30 30 30 F F F
17462024-02-07T18:55:13.805Z 17 340-359 12 12 12 23 23 23 F F F
17472024-02-07T18:55:13.805Z 18 360-379 15 15 15 31 31 31 F F F
17482024-02-07T18:55:13.805Z 19 380-399 9 9 9 20 20 20 F F F
17492024-02-07T18:55:13.805Z 20 400-419 14 14 14 30 30 30 F F F
17502024-02-07T18:55:13.805Z 21 420-439 14 14 14 30 30 30 F F F
17512024-02-07T18:55:13.805Z 22 440-459 11 11 11 22 22 22 F F F
17522024-02-07T18:55:13.805Z 23 460-479 12 12 12 24 24 24 F F F
17532024-02-07T18:55:13.805Z 24 480-499 14 14 14 29 29 29 F F F
17542024-02-07T18:55:13.806Z 25 500-519 14 14 14 29 29 29 F F F
17552024-02-07T18:55:13.806Z 26 520-539 12 12 12 27 27 27 F F F
17562024-02-07T18:55:13.806Z 27 540-559 10 10 10 21 21 21 F F F
17572024-02-07T18:55:13.806Z 28 560-579 10 10 10 21 21 21 F F F
17582024-02-07T18:55:13.806Z 29 580-599 4 4 4 9 9 9 F F F
17592024-02-07T18:55:13.806ZMax gen: 15, Max flush: 31
17602024-02-07T18:55:13.806Z
17612024-02-07T18:55:13.806ZBegin loop 8
17622024-02-07T18:55:13.806ZDownstairs to restart: 0
17632024-02-07T18:55:13.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.820835065Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199}
17642024-02-07T18:55:13.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821110058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199}
17652024-02-07T18:55:13.806Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821131675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199}
17662024-02-07T18:55:13.806Z{"msg":"Crucible 4351fb70-c022-42dd-9db6-99d3c24d3eb5 has session id: 0149d584-9e7e-4087-bc64-2ee9c82c3cc6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821158511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17672024-02-07T18:55:13.806ZCrucible runtime is spawned
17682024-02-07T18:55:13.806Z{"msg":"Upstairs opts: Upstairs UUID: 4351fb70-c022-42dd-9db6-99d3c24d3eb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821217003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17692024-02-07T18:55:13.806Z{"msg":"Crucible stats registered with UUID: 4351fb70-c022-42dd-9db6-99d3c24d3eb5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821248639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17702024-02-07T18:55:13.806Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821266147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199}
17712024-02-07T18:55:13.806Z{"msg":"4351fb70-c022-42dd-9db6-99d3c24d3eb5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821368283Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17722024-02-07T18:55:13.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.82139256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17732024-02-07T18:55:13.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821425415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17742024-02-07T18:55:13.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821458771Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17752024-02-07T18:55:13.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821491996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17762024-02-07T18:55:13.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821521882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17772024-02-07T18:55:13.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821552028Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17782024-02-07T18:55:13.807Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821581764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17792024-02-07T18:55:13.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.82161461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17802024-02-07T18:55:13.807Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821648295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17812024-02-07T18:55:13.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821697838Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17822024-02-07T18:55:13.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.821729954Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17832024-02-07T18:55:13.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.82176146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17842024-02-07T18:55:13.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.822417861Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17852024-02-07T18:55:13.807Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.822445597Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17862024-02-07T18:55:13.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.822461195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17872024-02-07T18:55:13.807Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.822476163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17882024-02-07T18:55:13.807Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823099199Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17892024-02-07T18:55:13.807Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823208354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17902024-02-07T18:55:13.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823320459Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17912024-02-07T18:55:13.807Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823378051Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17922024-02-07T18:55:13.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823598761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17932024-02-07T18:55:13.807Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823635506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17942024-02-07T18:55:13.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823657363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17952024-02-07T18:55:13.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:09.823737693Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17962024-02-07T18:55:13.807ZWait for a query_work_queue command to finish before sending IO
17972024-02-07T18:55:13.807Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.56953698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17982024-02-07T18:55:13.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.5696128Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
17992024-02-07T18:55:13.808Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569633997Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18002024-02-07T18:55:13.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569652295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18012024-02-07T18:55:13.808Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569669163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18022024-02-07T18:55:13.808Z{"msg":"[0]R flush_numbers[0..12]: [30, 30, 21, 31, 28, 31, 28, 14, 31, 24, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.56968708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18032024-02-07T18:55:13.808Z{"msg":"[0]R generation[0..12]: [14, 14, 10, 15, 13, 15, 13, 6, 15, 12, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569713326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18042024-02-07T18:55:13.808Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569732104Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18052024-02-07T18:55:13.808Z{"msg":"[1]R flush_numbers[0..12]: [30, 30, 21, 31, 28, 31, 28, 14, 31, 24, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569750312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18062024-02-07T18:55:13.808Z{"msg":"[1]R generation[0..12]: [14, 14, 10, 15, 13, 15, 13, 6, 15, 12, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569767769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18072024-02-07T18:55:13.808Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569785447Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18082024-02-07T18:55:13.808Z{"msg":"[2]R flush_numbers[0..12]: [30, 30, 21, 31, 28, 31, 28, 14, 31, 24, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569803104Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18092024-02-07T18:55:13.808Z{"msg":"[2]R generation[0..12]: [14, 14, 10, 15, 13, 15, 13, 6, 15, 12, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569820482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18102024-02-07T18:55:13.808Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.56983916Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18112024-02-07T18:55:13.808Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569856567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18122024-02-07T18:55:13.808Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569873705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18132024-02-07T18:55:13.808Z{"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569890233Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18142024-02-07T18:55:13.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.56990727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18152024-02-07T18:55:13.815Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569924098Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18162024-02-07T18:55:13.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569939106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18172024-02-07T18:55:13.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569955824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18182024-02-07T18:55:13.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569975031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18192024-02-07T18:55:13.815Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.569992249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18202024-02-07T18:55:13.815Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.570008097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18212024-02-07T18:55:13.815Z{"msg":"4351fb70-c022-42dd-9db6-99d3c24d3eb5 is now active with session: 0149d584-9e7e-4087-bc64-2ee9c82c3cc6","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18222024-02-07T18:55:13.815Z,"time":"2024-02-07T18:55:12.570024784Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18232024-02-07T18:55:13.815Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.57005974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199,"session_id":"0149d584-9e7e-4087-bc64-2ee9c82c3cc6"}
18242024-02-07T18:55:13.815Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:12.570076567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199}
18252024-02-07T18:55:13.815ZLoading write count information from file "/tmp/test_repair_verify.data"
18262024-02-07T18:55:13.815ZRun Repair workload
18272024-02-07T18:55:13.816Z01/30 Write block 242 len 40960 data: 3 4 4 3 3 4 4 3 3 3
18282024-02-07T18:55:13.816Z02/30 Read block 344 len 32768
18292024-02-07T18:55:13.816Z03/30 Read block 277 len 8192
18302024-02-07T18:55:13.816Z04/30 Write block 66 len 32768 data: 5 5 5 6 5 5 3 3
18312024-02-07T18:55:13.816Z05/30 Write block 464 len 16384 data: 3 3 3 2
18322024-02-07T18:55:13.816Z06/30 Write block 354 len 8192 data: 3 2
18332024-02-07T18:55:13.816Z07/30 Flush
18342024-02-07T18:55:13.816Z08/30 Write block 141 len 36864 data: 2 2 2 2 3 2 2 2 2
18352024-02-07T18:55:13.816Z09/30 Read block 472 len 24576
18362024-02-07T18:55:13.816Z10/30 Write block 586 len 28672 data: 3 3 3 3 2 3 3
18372024-02-07T18:55:13.816Z11/30 Read block 71 len 4096
18382024-02-07T18:55:13.816Z12/30 Read block 477 len 16384
18392024-02-07T18:55:13.816Z13/30 Read block 15 len 40960
18402024-02-07T18:55:13.816Z14/30 Write block 14 len 4096 data: 3
18412024-02-07T18:55:13.816Z15/30 Write block 360 len 20480 data: 4 4 4 3 3
18422024-02-07T18:55:13.816Z16/30 Read block 397 len 4096
18432024-02-07T18:55:13.816Z17/30 Read block 175 len 36864
18442024-02-07T18:55:13.816Z18/30 Read block 316 len 40960
18452024-02-07T18:55:13.816Z19/30 Read block 505 len 24576
18462024-02-07T18:55:13.816Z20/30 Read block 20 len 36864
18472024-02-07T18:55:13.816Z21/30 Write block 410 len 24576 data: 3 4 3 3 3 3
18482024-02-07T18:55:13.816Z22/30 Read block 317 len 4096
18492024-02-07T18:55:13.817Z23/30 Read block 519 len 20480
18502024-02-07T18:55:13.817Z24/30 Read block 85 len 32768
18512024-02-07T18:55:13.817Z25/30 Write block 14 len 16384 data: 4 3 3 3
18522024-02-07T18:55:13.817Z26/30 Read block 385 len 24576
18532024-02-07T18:55:13.817Z27/30 Write block 512 len 20480 data: 3 3 3 3 3
18542024-02-07T18:55:13.817Z28/30 Read block 86 len 32768
18552024-02-07T18:55:13.817Z29/30 Write block 327 len 20480 data: 2 2 2 2 3
18562024-02-07T18:55:13.817Z30/30 Write block 86 len 28672 data: 2 2 3 3 3 3 3
18572024-02-07T18:55:13.817Z----------------------------------------------------------------
18582024-02-07T18:55:13.817Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:29
18592024-02-07T18:55:13.817ZStates: Active Active Active
18602024-02-07T18:55:13.817ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18612024-02-07T18:55:13.817Z 3 Acked 1002 Read 8 Sent Done Done false
18622024-02-07T18:55:13.817Z 4 Acked 1003 Read 2 Sent Done Done false
18632024-02-07T18:55:13.817Z 5 Acked 1004 Write 8 Sent Done Done false
18642024-02-07T18:55:13.817Z 6 Acked 1005 Write 4 Sent Done Done false
18652024-02-07T18:55:13.817Z 7 Acked 1006 Write 2 Sent Done Done false
18662024-02-07T18:55:13.817Z 8 Acked 1007 Flush 0 Sent Done Done false
18672024-02-07T18:55:13.817Z 9 Acked 1008 Write 9 Sent Done Done false
18682024-02-07T18:55:13.817Z 10 Acked 1009 Read 6 Sent Done Done false
18692024-02-07T18:55:13.817Z 11 Acked 1010 Write 7 Sent Done Done false
18702024-02-07T18:55:13.818Z 12 Acked 1011 Read 1 Sent Done Done false
18712024-02-07T18:55:13.818Z 13 Acked 1012 Read 4 Sent Done Done false
18722024-02-07T18:55:13.818Z 14 Acked 1013 Read 10 Sent Done Done false
18732024-02-07T18:55:13.818Z 15 Acked 1014 Write 1 Sent Done Done false
18742024-02-07T18:55:13.818Z 16 Acked 1015 Write 5 Sent Done Done false
18752024-02-07T18:55:13.818Z 17 Acked 1016 Read 1 Sent Done Done false
18762024-02-07T18:55:13.818Z 18 Acked 1017 Read 9 Sent Done Done false
18772024-02-07T18:55:13.818Z 19 Acked 1018 Read 10 Sent Done Done false
18782024-02-07T18:55:13.818Z 20 Acked 1019 Read 6 Sent Done Done false
18792024-02-07T18:55:13.818Z 21 Acked 1020 Read 9 Sent Done Done false
18802024-02-07T18:55:13.818Z 22 Acked 1021 Write 6 Sent Done Done false
18812024-02-07T18:55:13.818Z 23 Acked 1022 Read 1 Sent Done Done false
18822024-02-07T18:55:13.818Z 24 Acked 1023 Read 5 Sent Done Done false
18832024-02-07T18:55:13.818Z 25 Acked 1024 Read 8 Sent Done Done false
18842024-02-07T18:55:13.818Z 26 Acked 1025 Write 4 Sent Done Done false
18852024-02-07T18:55:13.818Z 27 Acked 1026 Read 6 Sent Done Done false
18862024-02-07T18:55:13.818Z 28 Acked 1027 Write 5 Sent Done Done false
18872024-02-07T18:55:13.818Z 29 Acked 1028 Read 8 Sent Done Done false
18882024-02-07T18:55:13.818Z 30 Acked 1029 Write 5 Sent Sent Done false
18892024-02-07T18:55:13.818Z 31 Acked 1030 Write 7 Sent Sent Sent false
18902024-02-07T18:55:13.818Z STATES DS:0 DS:1 DS:2 TOTAL
18912024-02-07T18:55:13.818Z New 0 0 0 0
18922024-02-07T18:55:13.818Z Sent 29 2 1 32
18932024-02-07T18:55:13.818Z Done 0 27 28 55
18942024-02-07T18:55:13.818Z Skipped 0 0 0 0
18952024-02-07T18:55:13.819Z Error 0 0 0 0
18962024-02-07T18:55:13.819ZLast Flush: 1001 1007 1007
18972024-02-07T18:55:13.819ZDownstairs last five completed: 1001 1000
18982024-02-07T18:55:13.819ZUpstairs last five completed: 31 30 29 28 27
18992024-02-07T18:55:13.819Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:12.868492307Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1199}
19002024-02-07T18:55:13.819ZWrote out file "/tmp/test_repair_verify.data"
19012024-02-07T18:55:13.819Z
19022024-02-07T18:55:13.819ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
19032024-02-07T18:55:13.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19042024-02-07T18:55:13.819Z 0 000-019 14 14 14 30 30 30 F T T <---
19052024-02-07T18:55:13.819Z 1 020-039 14 14 14 30 30 30 F F F
19062024-02-07T18:55:13.819Z 2 040-059 10 10 10 21 21 21 F F F
19072024-02-07T18:55:13.819Z 3 060-079 15 16 16 31 33 33 F F F <---
19082024-02-07T18:55:13.819Z 4 080-099 13 13 13 28 28 28 F T T <---
19092024-02-07T18:55:13.819Z 5 100-119 15 15 15 31 31 31 F F F
19102024-02-07T18:55:13.819Z 6 120-139 13 13 13 28 28 28 F F F
19112024-02-07T18:55:13.819Z 7 140-159 6 6 6 14 14 14 F T T <---
19122024-02-07T18:55:13.819Z 8 160-179 15 15 15 31 31 31 F F F
19132024-02-07T18:55:13.819Z 9 180-199 12 12 12 24 24 24 F F F
19142024-02-07T18:55:13.819Z 10 200-219 15 15 15 31 31 31 F F F
19152024-02-07T18:55:13.825Z 11 220-239 15 15 15 31 31 31 F F F
19162024-02-07T18:55:13.825Z 12 240-259 16 16 16 32 32 32 F F F
19172024-02-07T18:55:13.825Z 13 260-279 6 6 6 16 16 16 F F F
19182024-02-07T18:55:13.825Z 14 280-299 15 15 15 31 31 31 F F F
19192024-02-07T18:55:13.825Z 15 300-319 12 12 12 24 24 24 F F F
19202024-02-07T18:55:13.825Z 16 320-339 14 14 14 30 30 30 F T T <---
19212024-02-07T18:55:13.825Z 17 340-359 12 16 16 23 33 33 F F F <---
19222024-02-07T18:55:13.825Z 18 360-379 15 15 15 31 31 31 F T T <---
19232024-02-07T18:55:13.825Z 19 380-399 9 9 9 20 20 20 F F F
19242024-02-07T18:55:13.826Z 20 400-419 14 14 14 30 30 30 F T T <---
19252024-02-07T18:55:13.826Z 21 420-439 14 14 14 30 30 30 F F F
19262024-02-07T18:55:13.826Z 22 440-459 11 11 11 22 22 22 F F F
19272024-02-07T18:55:13.826Z 23 460-479 12 16 16 24 33 33 F F F <---
19282024-02-07T18:55:13.826Z 24 480-499 14 14 14 29 29 29 F F F
19292024-02-07T18:55:13.826Z 25 500-519 14 14 14 29 29 29 F T T <---
19302024-02-07T18:55:13.826Z 26 520-539 12 12 12 27 27 27 F F F
19312024-02-07T18:55:13.826Z 27 540-559 10 10 10 21 21 21 F F F
19322024-02-07T18:55:13.826Z 28 560-579 10 10 10 21 21 21 F F F
19332024-02-07T18:55:13.826Z 29 580-599 4 4 4 9 9 9 F T T <---
19342024-02-07T18:55:13.826ZMax gen: 16, Max flush: 33
19352024-02-07T18:55:13.826ZError: Difference in extent metadata found!
19362024-02-07T18:55:13.826ZA Difference in extent metadata is expected here
19372024-02-07T18:55:13.826ZOn loop 8
19382024-02-07T18:55:13.826Z
19392024-02-07T18:55:13.826ZVerifying data now
19402024-02-07T18:55:18.208ZNo change after verify
19412024-02-07T18:55:18.208ZLoop: 8 Downstairs dump after verify (and repair):
19422024-02-07T18:55:18.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19432024-02-07T18:55:18.225Z 0 000-019 17 17 17 34 34 34 F F F
19442024-02-07T18:55:18.225Z 1 020-039 14 14 14 30 30 30 F F F
19452024-02-07T18:55:18.225Z 2 040-059 10 10 10 21 21 21 F F F
19462024-02-07T18:55:18.225Z 3 060-079 16 16 16 33 33 33 F F F
19472024-02-07T18:55:18.226Z 4 080-099 17 17 17 34 34 34 F F F
19482024-02-07T18:55:18.226Z 5 100-119 15 15 15 31 31 31 F F F
19492024-02-07T18:55:18.226Z 6 120-139 13 13 13 28 28 28 F F F
19502024-02-07T18:55:18.226Z 7 140-159 17 17 17 34 34 34 F F F
19512024-02-07T18:55:18.226Z 8 160-179 15 15 15 31 31 31 F F F
19522024-02-07T18:55:18.226Z 9 180-199 12 12 12 24 24 24 F F F
19532024-02-07T18:55:18.227Z 10 200-219 15 15 15 31 31 31 F F F
19542024-02-07T18:55:18.227Z 11 220-239 15 15 15 31 31 31 F F F
19552024-02-07T18:55:18.227Z 12 240-259 16 16 16 32 32 32 F F F
19562024-02-07T18:55:18.228Z 13 260-279 6 6 6 16 16 16 F F F
19572024-02-07T18:55:18.228Z 14 280-299 15 15 15 31 31 31 F F F
19582024-02-07T18:55:18.228Z 15 300-319 12 12 12 24 24 24 F F F
19592024-02-07T18:55:18.228Z 16 320-339 17 17 17 34 34 34 F F F
19602024-02-07T18:55:18.228Z 17 340-359 16 16 16 33 33 33 F F F
19612024-02-07T18:55:18.228Z 18 360-379 17 17 17 34 34 34 F F F
19622024-02-07T18:55:18.228Z 19 380-399 9 9 9 20 20 20 F F F
19632024-02-07T18:55:18.228Z 20 400-419 17 17 17 34 34 34 F F F
19642024-02-07T18:55:18.228Z 21 420-439 14 14 14 30 30 30 F F F
19652024-02-07T18:55:18.229Z 22 440-459 11 11 11 22 22 22 F F F
19662024-02-07T18:55:18.229Z 23 460-479 16 16 16 33 33 33 F F F
19672024-02-07T18:55:18.229Z 24 480-499 14 14 14 29 29 29 F F F
19682024-02-07T18:55:18.229Z 25 500-519 17 17 17 34 34 34 F F F
19692024-02-07T18:55:18.229Z 26 520-539 12 12 12 27 27 27 F F F
19702024-02-07T18:55:18.230Z 27 540-559 10 10 10 21 21 21 F F F
19712024-02-07T18:55:18.230Z 28 560-579 10 10 10 21 21 21 F F F
19722024-02-07T18:55:18.230Z 29 580-599 17 17 17 34 34 34 F F F
19732024-02-07T18:55:18.230ZMax gen: 17, Max flush: 34
19742024-02-07T18:55:18.230Z
19752024-02-07T18:55:18.230ZBegin loop 9
19762024-02-07T18:55:18.230ZDownstairs to restart: 0
19772024-02-07T18:55:18.259Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496124077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207}
19782024-02-07T18:55:18.259Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496375203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207}
19792024-02-07T18:55:18.259Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496412848Z","hostname":"Crucible runtime is spawned
19802024-02-07T18:55:18.259Zip-10-150-1-187.us-west-2.compute.internal","pid":1207}
19812024-02-07T18:55:18.259Z{"msg":"Crucible dc0399d5-74f6-49fd-9e9f-1673d95128dc has session id: 7e04749c-4426-462b-ba3f-e75940b6dc98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496445493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19822024-02-07T18:55:18.259Z{"msg":"Upstairs opts: Upstairs UUID: dc0399d5-74f6-49fd-9e9f-1673d95128dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496510334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19832024-02-07T18:55:18.259Z{"msg":"Crucible stats registered with UUID: dc0399d5-74f6-49fd-9e9f-1673d95128dc","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496548109Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19842024-02-07T18:55:18.259Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496572916Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207}
19852024-02-07T18:55:18.259Z{"msg":"dc0399d5-74f6-49fd-9e9f-1673d95128dc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496753612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19862024-02-07T18:55:18.259Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496790677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19872024-02-07T18:55:18.259Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496809074Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19882024-02-07T18:55:18.259Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.49683808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19892024-02-07T18:55:18.259Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496861317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19902024-02-07T18:55:18.259Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496889433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19912024-02-07T18:55:18.259Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496904591Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19922024-02-07T18:55:18.259Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.496993579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19932024-02-07T18:55:18.259Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497067589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19942024-02-07T18:55:18.259Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497086567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19952024-02-07T18:55:18.259Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497107794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19962024-02-07T18:55:18.260Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497217989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19972024-02-07T18:55:18.260Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497270772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19982024-02-07T18:55:18.260Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497901426Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
19992024-02-07T18:55:18.260Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497942061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20002024-02-07T18:55:18.260Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497957749Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20012024-02-07T18:55:18.260Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.497988705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20022024-02-07T18:55:18.260Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.498580534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20032024-02-07T18:55:18.260Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.4986905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20042024-02-07T18:55:18.260Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499111133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20052024-02-07T18:55:18.260Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499211979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20062024-02-07T18:55:18.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499267881Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20072024-02-07T18:55:18.260Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499304097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20082024-02-07T18:55:18.260Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499329533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20092024-02-07T18:55:18.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499362359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20102024-02-07T18:55:18.260Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499396794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20112024-02-07T18:55:18.260Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499432459Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20122024-02-07T18:55:18.260Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.499693614Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20132024-02-07T18:55:18.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500148982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20142024-02-07T18:55:18.260Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500182838Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20152024-02-07T18:55:18.260Z{"msg":"[0]R flush_numbers[0..12]: [34, 30, 21, 33, 34, 31, 28, 34, 31, 24, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500199426Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20162024-02-07T18:55:18.261Z{"msg":"[0]R generation[0..12]: [17, 14, 10, 16, 17, 15, 13, 17, 15, 12, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500221772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20172024-02-07T18:55:18.261Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.50023677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20182024-02-07T18:55:18.261Z{"msg":"[1]R flush_numbers[0..12]: [34, 30, 21, 33, 34, 31, 28, 34, 31, 24, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500257828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20192024-02-07T18:55:18.261Z{"msg":"[1]R generation[0..12]: [17, 14, 10, 16, 17, 15, 13, 17, 15, 12, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500272166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20202024-02-07T18:55:18.261Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500294003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20212024-02-07T18:55:18.261Z{"msg":"[2]R flush_numbers[0..12]: [34, 30, 21, 33, 34, 31, 28, 34, 31, 24, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500308661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20222024-02-07T18:55:18.261Z{"msg":"[2]R generation[0..12]: [17, 14, 10, 16, 17, 15, 13, 17, 15, 12, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500326898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20232024-02-07T18:55:18.261Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500341276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20242024-02-07T18:55:18.261Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500367723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20252024-02-07T18:55:18.261Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
20262024-02-07T18:55:18.261Zlevel":30,"time":"2024-02-07T18:55:17.500394659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20272024-02-07T18:55:18.261Z{"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500425345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20282024-02-07T18:55:18.261Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500450272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20292024-02-07T18:55:18.261Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500484817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20302024-02-07T18:55:18.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500508274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20312024-02-07T18:55:18.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.50053862Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20322024-02-07T18:55:18.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500563486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20332024-02-07T18:55:18.261Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500595582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20342024-02-07T18:55:18.261Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500619159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20352024-02-07T18:55:18.261Z{"msg":"dc0399d5-74f6-49fd-9e9f-1673d95128dc is now active with session: 7e04749c-4426-462b-ba3f-e75940b6dc98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500647675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20362024-02-07T18:55:18.261Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500669982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207,"session_id":"7e04749c-4426-462b-ba3f-e75940b6dc98"}
20372024-02-07T18:55:18.262Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:17.500696628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207}
20382024-02-07T18:55:18.262ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20392024-02-07T18:55:18.262ZLoading write count information from file "/tmp/test_repair_verify.data"
20402024-02-07T18:55:18.262ZRun Repair workload
20412024-02-07T18:55:18.262Z01/30 Flush
20422024-02-07T18:55:18.262Z02/30 Write block 440 len 20480 data: 3 2 2 2 2
20432024-02-07T18:55:18.262Z03/30 Read block 269 len 12288
20442024-02-07T18:55:18.262Z04/30 Read block 301 len 40960
20452024-02-07T18:55:18.262Z05/30 Flush
20462024-02-07T18:55:18.262Z06/30 Write block 334 len 32768 data: 2 2 2 2 2 3 3 3
20472024-02-07T18:55:18.262Z07/30 Write block 76 len 20480 data: 3 3 5 4 3
20482024-02-07T18:55:18.262Z08/30 Read block 425 len 28672
20492024-02-07T18:55:18.262Z09/30 Write block 425 len 28672 data: 4 4 4 5 4 3 3
20502024-02-07T18:55:18.262Z10/30 Read block 472 len 8192
20512024-02-07T18:55:18.262Z11/30 Write block 234 len 40960 data: 5 6 5 5 5 5 4 4 4 5
20522024-02-07T18:55:18.284Z12/30 Read block 109 len 32768
20532024-02-07T18:55:18.284Z13/30 Read block 373 len 16384
20542024-02-07T18:55:18.284Z14/30 Read block 201 len 20480
20552024-02-07T18:55:18.284Z15/30 Read block 404 len 12288
20562024-02-07T18:55:18.284Z16/30 Read block 548 len 36864
20572024-02-07T18:55:18.284Z17/30 Read block 287 len 12288
20582024-02-07T18:55:18.284Z18/30 Write block 146 len 20480 data: 3 3 3 3 2
20592024-02-07T18:55:18.284Z19/30 Write block 558 len 8192 data: 4 4
20602024-02-07T18:55:18.284Z20/30 Write block 591 len 16384 data: 4 4 3 3
20612024-02-07T18:55:18.284Z21/30 Read block 44 len 32768
20622024-02-07T18:55:18.284Z22/30 Flush
20632024-02-07T18:55:18.284Z23/30 Write block 542 len 4096 data: 3
20642024-02-07T18:55:18.284Z24/30 Read block 509 len 20480
20652024-02-07T18:55:18.284Z25/30 Flush
20662024-02-07T18:55:18.284Z26/30 Write block 392 len 28672 data: 2 3 3 4 4 4 4
20672024-02-07T18:55:18.284Z27/30 Read block 431 len 32768
20682024-02-07T18:55:18.284Z28/30 Write block 339 len 8192 data: 4 4
20692024-02-07T18:55:18.285Z29/30 Read block 383 len 36864
20702024-02-07T18:55:18.285Z30/30 Read block 43 len 28672
20712024-02-07T18:55:18.285Z----------------------------------------------------------------
20722024-02-07T18:55:18.285Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20732024-02-07T18:55:18.285ZStates: Active Active Active
20742024-02-07T18:55:18.285ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20752024-02-07T18:55:18.285Z 1 Acked 1000 Flush 0 Sent Done Done false
20762024-02-07T18:55:18.285Z 2 Acked 1001 Write 5 Sent Done Done false
20772024-02-07T18:55:18.285Z 3 Acked 1002 Read 3 Sent Done Done false
20782024-02-07T18:55:18.285Z 4 Acked 1003 Read 10 Sent Done Done false
20792024-02-07T18:55:18.285Z 5 Acked 1004 Flush 0 Sent Done Done false
20802024-02-07T18:55:18.285Z 6 Acked 1005 Write 8 Sent Done Done false
20812024-02-07T18:55:18.285Z 7 Acked 1006 Write 5 Sent Done Done false
20822024-02-07T18:55:18.285Z 8 Acked 1007 Read 7 Sent Done Done false
20832024-02-07T18:55:18.285Z 9 Acked 1008 Write 7 Sent Done Done false
20842024-02-07T18:55:18.285Z 10 Acked 1009 Read 2 Sent Done Done false
20852024-02-07T18:55:18.285Z 11 Acked 1010 Write 10 Sent Done Done false
20862024-02-07T18:55:18.285Z 12 Acked 1011 Read 8 Sent Done Done false
20872024-02-07T18:55:18.285Z 13 Acked 1012 Read 4 Sent Done Done false
20882024-02-07T18:55:18.285Z 14 Acked 1013 Read 5 Sent Done Done false
20892024-02-07T18:55:18.285Z 15 Acked 1014 Read 3 Sent Done Done false
20902024-02-07T18:55:18.285Z 16 Acked 1015 Read 9 Sent Done Done false
20912024-02-07T18:55:18.285Z 17 Acked 1016 Read 3 Sent Done Done false
20922024-02-07T18:55:18.286Z 18 Acked 1017 Write 5 Sent Done Done false
20932024-02-07T18:55:18.286Z 19 Acked 1018 Write 2 Sent Done Done false
20942024-02-07T18:55:18.286Z 20 Acked 1019 Write 4 Sent Done Done false
20952024-02-07T18:55:18.286Z 21 Acked 1020 Read 8 Sent Done Done false
20962024-02-07T18:55:18.286Z 22 Acked 1021 Flush 0 Sent Done Done false
20972024-02-07T18:55:18.286Z 23 Acked 1022 Write 1 Sent Done Done false
20982024-02-07T18:55:18.286Z 24 Acked 1023 Read 5 Sent Done Done false
20992024-02-07T18:55:18.286Z 25 Acked 1024 Flush 0 Sent Done Done false
21002024-02-07T18:55:18.286Z 26 Acked 1025 Write 7 Sent Done Done false
21012024-02-07T18:55:18.286Z 27 Acked 1026 Read 8 Sent Done Done false
21022024-02-07T18:55:18.286Z 28 Acked 1027 Write 2 Sent Done Done false
21032024-02-07T18:55:18.286Z 29 Acked 1028 Read 9 Sent Done Done false
21042024-02-07T18:55:18.286Z 30 Acked 1029 Read 7 Sent Done Done false
21052024-02-07T18:55:18.286Z STATES DS:0 DS:1 DS:2 TOTAL
21062024-02-07T18:55:18.286Z New 0 0 0 0
21072024-02-07T18:55:18.286Z Sent 30 0 0 30
21082024-02-07T18:55:18.286Z Done 0 30 30 60
21092024-02-07T18:55:18.286Z Skipped 0 0 0 0
21102024-02-07T18:55:18.286Z Error 0 0 0 0
21112024-02-07T18:55:18.286ZLast Flush: 0 1024 1024
21122024-02-07T18:55:18.286ZDownstairs last five completed:
21132024-02-07T18:55:18.286ZUpstairs last five completed: 30 29 28 27 26
21142024-02-07T18:55:18.286Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:17.534202187Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1207}
21152024-02-07T18:55:18.286ZWrote out file "/tmp/test_repair_verify.data"
21162024-02-07T18:55:18.287Z
21172024-02-07T18:55:18.297ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21182024-02-07T18:55:18.297ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21192024-02-07T18:55:18.297Z 0 000-019 17 17 17 34 34 34 F F F
21202024-02-07T18:55:18.297Z 1 020-039 14 14 14 30 30 30 F F F
21212024-02-07T18:55:18.297Z 2 040-059 10 10 10 21 21 21 F F F
21222024-02-07T18:55:18.297Z 3 060-079 16 18 18 33 37 37 F F F <---
21232024-02-07T18:55:18.297Z 4 080-099 17 18 18 34 37 37 F F F <---
21242024-02-07T18:55:18.297Z 5 100-119 15 15 15 31 31 31 F F F
21252024-02-07T18:55:18.297Z 6 120-139 13 13 13 28 28 28 F F F
21262024-02-07T18:55:18.298Z 7 140-159 17 18 18 34 37 37 F F F <---
21272024-02-07T18:55:18.298Z 8 160-179 15 15 15 31 31 31 F F F
21282024-02-07T18:55:18.298Z 9 180-199 12 12 12 24 24 24 F F F
21292024-02-07T18:55:18.298Z 10 200-219 15 15 15 31 31 31 F F F
21302024-02-07T18:55:18.298Z 11 220-239 15 18 18 31 37 37 F F F <---
21312024-02-07T18:55:18.298Z 12 240-259 16 18 18 32 37 37 F F F <---
21322024-02-07T18:55:18.298Z 13 260-279 6 6 6 16 16 16 F F F
21332024-02-07T18:55:18.298Z 14 280-299 15 15 15 31 31 31 F F F
21342024-02-07T18:55:18.298Z 15 300-319 12 12 12 24 24 24 F F F
21352024-02-07T18:55:18.298Z 16 320-339 17 18 18 34 37 37 F T T <---
21362024-02-07T18:55:18.298Z 17 340-359 16 18 18 33 37 37 F T T <---
21372024-02-07T18:55:18.298Z 18 360-379 17 17 17 34 34 34 F F F
21382024-02-07T18:55:18.298Z 19 380-399 9 9 9 20 20 20 F T T <---
21392024-02-07T18:55:18.299Z 20 400-419 17 17 17 34 34 34 F F F
21402024-02-07T18:55:18.299Z 21 420-439 14 18 18 30 37 37 F F F <---
21412024-02-07T18:55:18.299Z 22 440-459 11 18 18 22 36 36 F F F <---
21422024-02-07T18:55:18.299Z 23 460-479 16 16 16 33 33 33 F F F
21432024-02-07T18:55:18.299Z 24 480-499 14 14 14 29 29 29 F F F
21442024-02-07T18:55:18.299Z 25 500-519 17 17 17 34 34 34 F F F
21452024-02-07T18:55:18.299Z 26 520-539 12 12 12 27 27 27 F F F
21462024-02-07T18:55:18.299Z 27 540-559 10 18 18 21 38 38 F F F <---
21472024-02-07T18:55:18.299Z 28 560-579 10 10 10 21 21 21 F F F
21482024-02-07T18:55:18.299Z 29 580-599 17 18 18 34 37 37 F F F <---
21492024-02-07T18:55:18.299ZMax gen: 18, Max flush: 38
21502024-02-07T18:55:18.299ZError: Difference in extent metadata found!
21512024-02-07T18:55:18.299ZA Difference in extent metadata is expected here
21522024-02-07T18:55:18.300ZOn loop 9
21532024-02-07T18:55:18.300Z
21542024-02-07T18:55:18.306ZVerifying data now
21552024-02-07T18:55:22.899ZNo change after verify
21562024-02-07T18:55:22.899ZLoop: 9 Downstairs dump after verify (and repair):
21572024-02-07T18:55:23.326ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21582024-02-07T18:55:23.326Z 0 000-019 17 17 17 34 34 34 F F F
21592024-02-07T18:55:23.326Z 1 020-039 14 14 14 30 30 30 F F F
21602024-02-07T18:55:23.326Z 2 040-059 10 10 10 21 21 21 F F F
21612024-02-07T18:55:23.326Z 3 060-079 18 18 18 37 37 37 F F F
21622024-02-07T18:55:23.327Z 4 080-099 18 18 18 37 37 37 F F F
21632024-02-07T18:55:23.327Z 5 100-119 15 15 15 31 31 31 F F F
21642024-02-07T18:55:23.327Z 6 120-139 13 13 13 28 28 28 F F F
21652024-02-07T18:55:23.327Z 7 140-159 18 18 18 37 37 37 F F F
21662024-02-07T18:55:23.327Z 8 160-179 15 15 15 31 31 31 F F F
21672024-02-07T18:55:23.327Z 9 180-199 12 12 12 24 24 24 F F F
21682024-02-07T18:55:23.328Z 10 200-219 15 15 15 31 31 31 F F F
21692024-02-07T18:55:23.356Z 11 220-239 18 18 18 37 37 37 F F F
21702024-02-07T18:55:23.356Z 12 240-259 18 18 18 37 37 37 F F F
21712024-02-07T18:55:23.357Z 13 260-279 6 6 6 16 16 16 F F F
21722024-02-07T18:55:23.357Z 14 280-299 15 15 15 31 31 31 F F F
21732024-02-07T18:55:23.357Z 15 300-319 12 12 12 24 24 24 F F F
21742024-02-07T18:55:23.357Z 16 320-339 19 19 19 39 39 39 F F F
21752024-02-07T18:55:23.357Z 17 340-359 19 19 19 39 39 39 F F F
21762024-02-07T18:55:23.357Z 18 360-379 17 17 17 34 34 34 F F F
21772024-02-07T18:55:23.358Z 19 380-399 19 19 19 39 39 39 F F F
21782024-02-07T18:55:23.358Z 20 400-419 17 17 17 34 34 34 F F F
21792024-02-07T18:55:23.358Z 21 420-439 18 18 18 37 37 37 F F F
21802024-02-07T18:55:23.358Z 22 440-459 18 18 18 36 36 36 F F F
21812024-02-07T18:55:23.359Z 23 460-479 16 16 16 33 33 33 F F F
21822024-02-07T18:55:23.359Z 24 480-499 14 14 14 29 29 29 F F F
21832024-02-07T18:55:23.359Z 25 500-519 17 17 17 34 34 34 F F F
21842024-02-07T18:55:23.359Z 26 520-539 12 12 12 27 27 27 F F F
21852024-02-07T18:55:23.359Z 27 540-559 18 18 18 38 38 38 F F F
21862024-02-07T18:55:23.360Z 28 560-579 10 10 10 21 21 21 F F F
21872024-02-07T18:55:23.360Z 29 580-599 18 18 18 37 37 37 F F F
21882024-02-07T18:55:23.360ZMax gen: 19, Max flush: 39
21892024-02-07T18:55:23.360Z
21902024-02-07T18:55:23.360ZBegin loop 10
21912024-02-07T18:55:23.361ZDownstairs to restart: 2
21922024-02-07T18:55:23.361Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.204948345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215}
21932024-02-07T18:55:23.361Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205273031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215}
21942024-02-07T18:55:23.361Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205310046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215Crucible runtime is spawned
21952024-02-07T18:55:23.362Z}
21962024-02-07T18:55:23.362Z{"msg":"Crucible f4d02ebe-986c-49de-aab7-7b6c143e9c98 has session id: fbf81b2f-4fb8-4d47-a3af-446467a272b4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205345021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
21972024-02-07T18:55:23.362Z{"msg":"Upstairs opts: Upstairs UUID: f4d02ebe-986c-49de-aab7-7b6c143e9c98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205387356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
21982024-02-07T18:55:23.362Z{"msg":"Crucible stats registered with UUID: f4d02ebe-986c-49de-aab7-7b6c143e9c98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205405053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
21992024-02-07T18:55:23.363Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205418562Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215}
22002024-02-07T18:55:23.363Z{"msg":"f4d02ebe-986c-49de-aab7-7b6c143e9c98 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205631803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22012024-02-07T18:55:23.363Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205659869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22022024-02-07T18:55:23.363Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205675677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22032024-02-07T18:55:23.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205689515Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22042024-02-07T18:55:23.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205703743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22052024-02-07T18:55:23.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205717351Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22062024-02-07T18:55:23.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205754686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22072024-02-07T18:55:23.364Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205775363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22082024-02-07T18:55:23.365Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205807059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22092024-02-07T18:55:23.365Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.205884748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22102024-02-07T18:55:23.365Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206003643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22112024-02-07T18:55:23.365Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206040258Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22122024-02-07T18:55:23.366Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206067064Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22132024-02-07T18:55:23.366Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206755581Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22142024-02-07T18:55:23.366Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206795045Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22152024-02-07T18:55:23.366Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206820772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22162024-02-07T18:55:23.367Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.206873605Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22172024-02-07T18:55:23.367Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.207442728Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22182024-02-07T18:55:23.367Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.207580799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22192024-02-07T18:55:23.367Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.207609805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22202024-02-07T18:55:23.368Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.207625903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22212024-02-07T18:55:23.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208006052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22222024-02-07T18:55:23.368Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208042787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22232024-02-07T18:55:23.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208069013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22242024-02-07T18:55:23.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208113497Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22252024-02-07T18:55:23.369Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208150742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22262024-02-07T18:55:23.369Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208185907Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22272024-02-07T18:55:23.369Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208372852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22282024-02-07T18:55:23.369Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208821831Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22292024-02-07T18:55:23.369Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208857306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22302024-02-07T18:55:23.370Z{"msg":"[0]R flush_numbers[0..12]: [34, 30, 21, 37, 37, 31, 28, 37, 31, 24, 31, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208873044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22312024-02-07T18:55:23.370Z{"msg":"[0]R generation[0..12]: [17, 14, 10, 18, 18, 15, 13, 18, 15, 12, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208888242Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22322024-02-07T18:55:23.370Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.20890345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22332024-02-07T18:55:23.370Z{"msg":"[1]R flush_numbers[0..12]: [34, 30, 21, 37, 37, 31, 28, 37, 31, 24, 31, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208918098Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22342024-02-07T18:55:23.371Z{"msg":"[1]R generation[0..12]: [17, 14, 10, 18, 18, 15, 13, 18, 15, 12, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208939335Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22352024-02-07T18:55:23.371Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208953993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22362024-02-07T18:55:23.371Z{"msg":"[2]R flush_numbers[0..12]: [34, 30, 21, 37, 37, 31, 28, 37, 31, 24, 31, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208968602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22372024-02-07T18:55:23.371Z{"msg":"[2]R generation[0..12]: [17, 14, 10, 18, 18, 15, 13, 18, 15, 12, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.208983269Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22382024-02-07T18:55:23.371Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209004407Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22392024-02-07T18:55:23.372Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209018655Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22402024-02-07T18:55:23.372Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209032833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
22412024-02-07T18:55:23.372Z,"":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22422024-02-07T18:55:23.372Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209048661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22432024-02-07T18:55:23.372Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209069458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22442024-02-07T18:55:23.373Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209083746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22452024-02-07T18:55:23.373Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209095984Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22462024-02-07T18:55:23.373Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209109312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22472024-02-07T18:55:23.373Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.20912951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22482024-02-07T18:55:23.374Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209143378Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22492024-02-07T18:55:23.374Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209156226Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22502024-02-07T18:55:23.374Z{"msg":"f4d02ebe-986c-49de-aab7-7b6c143e9c98 is now active with session: fbf81b2f-4fb8-4d47-a3af-446467a272b4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209175743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22512024-02-07T18:55:23.374Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.209188172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215,"session_id":"fbf81b2f-4fb8-4d47-a3af-446467a272b4"}
22522024-02-07T18:55:23.374Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:22.20920044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215}
22532024-02-07T18:55:23.375ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22542024-02-07T18:55:23.375ZLoading write count information from file "/tmp/test_repair_verify.data"
22552024-02-07T18:55:23.375ZRun Repair workload
22562024-02-07T18:55:23.375Z01/30 Flush
22572024-02-07T18:55:23.376Z02/30 Write block 176 len 12288 data: 4 4 3
22582024-02-07T18:55:23.376Z03/30 Write block 161 len 36864 data: 4 5 5 6 6 6 6 6 6
22592024-02-07T18:55:23.376Z04/30 Read block 349 len 12288
22602024-02-07T18:55:23.376Z05/30 Read block 196 len 4096
22612024-02-07T18:55:23.377Z06/30 Write block 180 len 4096 data: 4
22622024-02-07T18:55:23.377Z07/30 Read block 195 len 40960
22632024-02-07T18:55:23.377Z08/30 Flush
22642024-02-07T18:55:23.377Z09/30 Write block 118 len 12288 data: 4 5 4
22652024-02-07T18:55:23.377Z10/30 Read block 8 len 16384
22662024-02-07T18:55:23.378Z11/30 Read block 469 len 40960
22672024-02-07T18:55:23.378Z12/30 Read block 430 len 28672
22682024-02-07T18:55:23.378Z13/30 Write block 290 len 8192 data: 4 2
22692024-02-07T18:55:23.464Z14/30 Write block 389 len 24576 data: 2 2 2 3 4 4
22702024-02-07T18:55:23.464Z15/30 Read block 429 len 36864
22712024-02-07T18:55:23.465Z16/30 Write block 255 len 40960 data: 2 2 2 2 2 2 2 2 2 2
22722024-02-07T18:55:23.465Z17/30 Write block 321 len 16384 data: 3 3 3 2
22732024-02-07T18:55:23.465Z18/30 Read block 542 len 36864
22742024-02-07T18:55:23.465Z19/30 Flush
22752024-02-07T18:55:23.465Z20/30 Write block 549 len 32768 data: 2 2 2 3 3 3 3 4
22762024-02-07T18:55:23.465Z21/30 Read block 493 len 20480
22772024-02-07T18:55:23.466Z22/30 Read block 434 len 40960
22782024-02-07T18:55:23.466Z23/30 Read block 128 len 40960
22792024-02-07T18:55:23.466Z24/30 Write block 338 len 8192 data: 3 5
22802024-02-07T18:55:23.466Z25/30 Read block 517 len 36864
22812024-02-07T18:55:23.466Z26/30 Read block 475 len 32768
22822024-02-07T18:55:23.466Z27/30 Read block 4 len 32768
22832024-02-07T18:55:23.467Z28/30 Read block 110 len 16384
22842024-02-07T18:55:23.467Z29/30 Write block 367 len 20480 data: 3 3 3 2 2
22852024-02-07T18:55:23.467Z30/30 Read block 91 len 8192
22862024-02-07T18:55:23.467Z----------------------------------------------------------------
22872024-02-07T18:55:23.467Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:29
22882024-02-07T18:55:23.467ZStates: Active Active Active
22892024-02-07T18:55:23.468ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22902024-02-07T18:55:23.468Z 2 Acked 1001 Write 3 Done Done Done false
22912024-02-07T18:55:23.468Z 3 Acked 1002 Write 9 Done Done Done false
22922024-02-07T18:55:23.468Z 4 Acked 1003 Read 3 Done Done Done false
22932024-02-07T18:55:23.468Z 5 Acked 1004 Read 1 Done Done Sent false
22942024-02-07T18:55:23.468Z 6 Acked 1005 Write 1 Done Done Sent false
22952024-02-07T18:55:23.468Z 7 Acked 1006 Read 10 Done Done Sent false
22962024-02-07T18:55:23.468Z 8 Acked 1007 Flush 0 Done Done Sent false
22972024-02-07T18:55:23.469Z 9 Acked 1008 Write 3 Done Done Sent false
22982024-02-07T18:55:23.469Z 10 Acked 1009 Read 4 Done Done Sent false
22992024-02-07T18:55:23.469Z 11 Acked 1010 Read 10 Done Done Sent false
23002024-02-07T18:55:23.469Z 12 Acked 1011 Read 7 Done Done Sent false
23012024-02-07T18:55:23.469Z 13 Acked 1012 Write 2 Done Done Sent false
23022024-02-07T18:55:23.469Z 14 Acked 1013 Write 6 Done Done Sent false
23032024-02-07T18:55:23.469Z 15 Acked 1014 Read 9 Done Done Sent false
23042024-02-07T18:55:23.469Z 16 Acked 1015 Write 10 Done Done Sent false
23052024-02-07T18:55:23.469Z 17 Acked 1016 Write 4 Done Done Sent false
23062024-02-07T18:55:23.469Z 18 Acked 1017 Read 9 Done Done Sent false
23072024-02-07T18:55:23.470Z 19 Acked 1018 Flush 0 Done Done Sent false
23082024-02-07T18:55:23.470Z 20 Acked 1019 Write 8 Done Done Sent false
23092024-02-07T18:55:23.470Z 21 Acked 1020 Read 5 Done Done Sent false
23102024-02-07T18:55:23.470Z 22 Acked 1021 Read 10 Done Done Sent false
23112024-02-07T18:55:23.470Z 23 Acked 1022 Read 10 Done Done Sent false
23122024-02-07T18:55:23.470Z 24 Acked 1023 Write 2 Done Done Sent false
23132024-02-07T18:55:23.470Z 25 Acked 1024 Read 9 Done Done Sent false
23142024-02-07T18:55:23.470Z 26 Acked 1025 Read 8 Done Done Sent false
23152024-02-07T18:55:23.470Z 27 Acked 1026 Read 8 Done Done Sent false
23162024-02-07T18:55:23.470Z 28 Acked 1027 Read 4 Done Done Sent false
23172024-02-07T18:55:23.470Z 29 Acked 1028 Write 5 Done Done Sent false
23182024-02-07T18:55:23.471Z 30 Acked 1029 Read 2 Done Sent Sent false
23192024-02-07T18:55:23.471Z STATES DS:0 DS:1 DS:2 TOTAL
23202024-02-07T18:55:23.471Z New 0 0 0 0
23212024-02-07T18:55:23.471Z Sent 0 1 26 27
23222024-02-07T18:55:23.471Z Done 29 28 3 60
23232024-02-07T18:55:23.471Z Skipped 0 0 0 0
23242024-02-07T18:55:23.471Z Error 0 0 0 0
23252024-02-07T18:55:23.471ZLast Flush: 1018 1018 1000
23262024-02-07T18:55:23.471ZDownstairs last five completed: 1000
23272024-02-07T18:55:23.471ZUpstairs last five completed: 30 29 28 27 26
23282024-02-07T18:55:23.471Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:22.241315087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1215}
23292024-02-07T18:55:23.472ZWrote out file "/tmp/test_repair_verify.data"
23302024-02-07T18:55:23.472Z
23312024-02-07T18:55:23.472ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23322024-02-07T18:55:23.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23332024-02-07T18:55:23.472Z 0 000-019 17 17 17 34 34 34 F F F
23342024-02-07T18:55:23.472Z 1 020-039 14 14 14 30 30 30 F F F
23352024-02-07T18:55:23.472Z 2 040-059 10 10 10 21 21 21 F F F
23362024-02-07T18:55:23.472Z 3 060-079 18 18 18 37 37 37 F F F
23372024-02-07T18:55:23.472Z 4 080-099 18 18 18 37 37 37 F F F
23382024-02-07T18:55:23.472Z 5 100-119 20 20 15 42 42 31 F F F <---
23392024-02-07T18:55:23.473Z 6 120-139 20 20 13 42 42 28 F F F <---
23402024-02-07T18:55:23.473Z 7 140-159 18 18 18 37 37 37 F F F
23412024-02-07T18:55:23.473Z 8 160-179 20 20 15 41 41 31 F F T <---
23422024-02-07T18:55:23.473Z 9 180-199 20 20 12 41 41 24 F F F <---
23432024-02-07T18:55:23.473Z 10 200-219 15 15 15 31 31 31 F F F
23442024-02-07T18:55:23.474Z 11 220-239 18 18 18 37 37 37 F F F
23452024-02-07T18:55:23.474Z 12 240-259 20 20 18 42 42 37 F F F <---
23462024-02-07T18:55:23.474Z 13 260-279 20 20 6 42 42 16 F F F <---
23472024-02-07T18:55:23.474Z 14 280-299 20 20 15 42 42 31 F F F <---
23482024-02-07T18:55:23.474Z 15 300-319 12 12 12 24 24 24 F F F
23492024-02-07T18:55:23.474Z 16 320-339 20 20 19 42 42 39 T T F <---
23502024-02-07T18:55:23.474Z 17 340-359 19 19 19 39 39 39 F F F
23512024-02-07T18:55:23.475Z 18 360-379 17 17 17 34 34 34 T T F <---
23522024-02-07T18:55:23.475Z 19 380-399 20 20 19 42 42 39 F F F <---
23532024-02-07T18:55:23.475Z 20 400-419 17 17 17 34 34 34 F F F
23542024-02-07T18:55:23.475Z 21 420-439 18 18 18 37 37 37 F F F
23552024-02-07T18:55:23.475Z 22 440-459 18 18 18 36 36 36 F F F
23562024-02-07T18:55:23.475Z 23 460-479 16 16 16 33 33 33 F F F
23572024-02-07T18:55:23.475Z 24 480-499 14 14 14 29 29 29 F F F
23582024-02-07T18:55:23.476Z 25 500-519 17 17 17 34 34 34 F F F
23592024-02-07T18:55:23.476Z 26 520-539 12 12 12 27 27 27 F F F
23602024-02-07T18:55:23.476Z 27 540-559 18 18 18 38 38 38 T T F <---
23612024-02-07T18:55:23.476Z 28 560-579 10 10 10 21 21 21 F F F
23622024-02-07T18:55:23.476Z 29 580-599 18 18 18 37 37 37 F F F
23632024-02-07T18:55:23.476ZMax gen: 20, Max flush: 42
23642024-02-07T18:55:23.476ZError: Difference in extent metadata found!
23652024-02-07T18:55:23.477ZA Difference in extent metadata is expected here
23662024-02-07T18:55:23.477ZOn loop 10
23672024-02-07T18:55:23.477Z
23682024-02-07T18:55:23.477ZVerifying data now
23692024-02-07T18:55:27.573ZNo change after verify
23702024-02-07T18:55:27.577ZLoop: 10 Downstairs dump after verify (and repair):
23712024-02-07T18:55:27.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23722024-02-07T18:55:27.583Z 0 000-019 17 17 17 34 34 34 F F F
23732024-02-07T18:55:27.583Z 1 020-039 14 14 14 30 30 30 F F F
23742024-02-07T18:55:27.583Z 2 040-059 10 10 10 21 21 21 F F F
23752024-02-07T18:55:27.583Z 3 060-079 18 18 18 37 37 37 F F F
23762024-02-07T18:55:27.583Z 4 080-099 18 18 18 37 37 37 F F F
23772024-02-07T18:55:27.583Z 5 100-119 20 20 20 42 42 42 F F F
23782024-02-07T18:55:27.583Z 6 120-139 20 20 20 42 42 42 F F F
23792024-02-07T18:55:27.583Z 7 140-159 18 18 18 37 37 37 F F F
23802024-02-07T18:55:27.583Z 8 160-179 20 20 20 41 41 41 F F F
23812024-02-07T18:55:27.584Z 9 180-199 20 20 20 41 41 41 F F F
23822024-02-07T18:55:27.584Z 10 200-219 15 15 15 31 31 31 F F F
23832024-02-07T18:55:27.584Z 11 220-239 18 18 18 37 37 37 F F F
23842024-02-07T18:55:27.584Z 12 240-259 20 20 20 42 42 42 F F F
23852024-02-07T18:55:27.584Z 13 260-279 20 20 20 42 42 42 F F F
23862024-02-07T18:55:27.584Z 14 280-299 20 20 20 42 42 42 F F F
23872024-02-07T18:55:27.584Z 15 300-319 12 12 12 24 24 24 F F F
23882024-02-07T18:55:27.584Z 16 320-339 21 21 21 43 43 43 F F F
23892024-02-07T18:55:27.584Z 17 340-359 19 19 19 39 39 39 F F F
23902024-02-07T18:55:27.584Z 18 360-379 21 21 21 43 43 43 F F F
23912024-02-07T18:55:27.584Z 19 380-399 20 20 20 42 42 42 F F F
23922024-02-07T18:55:27.584Z 20 400-419 17 17 17 34 34 34 F F F
23932024-02-07T18:55:27.585Z 21 420-439 18 18 18 37 37 37 F F F
23942024-02-07T18:55:27.585Z 22 440-459 18 18 18 36 36 36 F F F
23952024-02-07T18:55:27.585Z 23 460-479 16 16 16 33 33 33 F F F
23962024-02-07T18:55:27.585Z 24 480-499 14 14 14 29 29 29 F F F
23972024-02-07T18:55:27.585Z 25 500-519 17 17 17 34 34 34 F F F
23982024-02-07T18:55:27.585Z 26 520-539 12 12 12 27 27 27 F F F
23992024-02-07T18:55:27.585Z 27 540-559 21 21 21 43 43 43 F F F
24002024-02-07T18:55:27.585Z 28 560-579 10 10 10 21 21 21 F F F
24012024-02-07T18:55:27.585Z 29 580-599 18 18 18 37 37 37 F F F
24022024-02-07T18:55:27.585ZMax gen: 21, Max flush: 43
24032024-02-07T18:55:27.590Z
24042024-02-07T18:55:27.590ZBegin loop 11
24052024-02-07T18:55:27.590ZDownstairs to restart: 2
24062024-02-07T18:55:27.602Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.870516494Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223}
24072024-02-07T18:55:27.603Z{Crucible runtime is spawned
24082024-02-07T18:55:27.603Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.87091478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223}
24092024-02-07T18:55:27.607Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.870949625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223}
24102024-02-07T18:55:27.607Z{"msg":"Crucible 4b0cce12-6efa-4e08-8946-74e817f4f381 has session id: 35c5f19d-3d3e-44cc-ad12-8cabd3abc475","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.870979871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24112024-02-07T18:55:27.607Z{"msg":"Upstairs opts: Upstairs UUID: 4b0cce12-6efa-4e08-8946-74e817f4f381, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.87106311Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24122024-02-07T18:55:27.607Z{"msg":"Crucible stats registered with UUID: 4b0cce12-6efa-4e08-8946-74e817f4f381","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871107524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24132024-02-07T18:55:27.607Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871147719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223}
24142024-02-07T18:55:27.607Z{"msg":"4b0cce12-6efa-4e08-8946-74e817f4f381 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871205451Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24152024-02-07T18:55:27.607Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871224748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24162024-02-07T18:55:27.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871264013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24172024-02-07T18:55:27.607Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871288Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24182024-02-07T18:55:27.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871325394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24192024-02-07T18:55:27.607Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871344982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24202024-02-07T18:55:27.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871384546Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24212024-02-07T18:55:27.607Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871416302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24222024-02-07T18:55:27.607Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871470005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24232024-02-07T18:55:27.607Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.87150469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24242024-02-07T18:55:27.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871545505Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24252024-02-07T18:55:27.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871635013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24262024-02-07T18:55:27.607Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:26.871689065Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24272024-02-07T18:55:27.607Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:26.871710292Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24282024-02-07T18:55:27.608Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:26.871736789Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24292024-02-07T18:55:27.608Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871751087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24302024-02-07T18:55:27.608Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871765405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24312024-02-07T18:55:27.608Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.871890938Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24322024-02-07T18:55:27.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.87216985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24332024-02-07T18:55:27.608Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.872196826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24342024-02-07T18:55:27.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.872212544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24352024-02-07T18:55:27.608Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.87224238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24362024-02-07T18:55:27.608Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.872804544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24372024-02-07T18:55:27.608Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.872967672Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24382024-02-07T18:55:27.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.873418591Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24392024-02-07T18:55:27.608Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.873451957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24402024-02-07T18:55:27.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.873467435Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24412024-02-07T18:55:27.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.873558532Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24422024-02-07T18:55:27.608Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.873598547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24432024-02-07T18:55:27.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:26.873625993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24442024-02-07T18:55:28.635Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.873534525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24452024-02-07T18:55:28.651Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.873684054Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24462024-02-07T18:55:28.651Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.874488976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24472024-02-07T18:55:28.651Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.874526951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24482024-02-07T18:55:28.651Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875043321Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24492024-02-07T18:55:28.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875621223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24502024-02-07T18:55:28.651Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875677025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24512024-02-07T18:55:28.651Z{"msg":"[0]R flush_numbers[0..12]: [34, 30, 21, 37, 37, 42, 42, 37, 41, 41, 31, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875718889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24522024-02-07T18:55:28.651Z{"msg":"[0]R generation[0..12]: [17, 14, 10, 18, 18, 20, 20, 18, 20, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875738627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24532024-02-07T18:55:28.651Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875767263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24542024-02-07T18:55:28.651Z{"msg":"[1]R flush_numbers[0..12]: [34, 30, 21, 37, 37, 42, 42, 37, 41, 41, 31, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.87578628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24552024-02-07T18:55:28.652Z{"msg":"[1]R generation[0..12]: [17, 14, 10, 18, 18, 20, 20, 18, 20, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875804468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24562024-02-07T18:55:28.652Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
24572024-02-07T18:55:28.652Z,"time":"2024-02-07T18:55:27.875822625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24582024-02-07T18:55:28.652Z{"msg":"[2]R flush_numbers[0..12]: [34, 30, 21, 37, 37, 42, 42, 37, 41, 41, 31, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875849182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24592024-02-07T18:55:28.652Z{"msg":"[2]R generation[0..12]: [17, 14, 10, 18, 18, 20, 20, 18, 20, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875879298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24602024-02-07T18:55:28.652Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875926831Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24612024-02-07T18:55:28.652Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875951218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24622024-02-07T18:55:28.652Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.875979124Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24632024-02-07T18:55:28.652Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876020498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24642024-02-07T18:55:28.652Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876038166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24652024-02-07T18:55:28.652Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876065003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24662024-02-07T18:55:28.652Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.87608111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24672024-02-07T18:55:28.652Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876106347Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24682024-02-07T18:55:28.652Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876123465Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24692024-02-07T18:55:28.653Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876139902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24702024-02-07T18:55:28.653Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.87615521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24712024-02-07T18:55:28.653Z{"msg":"4b0cce12-6efa-4e08-8946-74e817f4f381 is now active with session: 35c5f19d-3d3e-44cc-ad12-8cabd3abc475","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876171958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24722024-02-07T18:55:28.653Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876186946Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223,"session_id":"35c5f19d-3d3e-44cc-ad12-8cabd3abc475"}
24732024-02-07T18:55:28.653Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:27.876210543Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223}
24742024-02-07T18:55:28.653ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24752024-02-07T18:55:28.653ZLoading write count information from file "/tmp/test_repair_verify.data"
24762024-02-07T18:55:28.653ZRun Repair workload
24772024-02-07T18:55:28.653Z01/30 Flush
24782024-02-07T18:55:28.653Z02/30 Write block 215 len 40960 data: 2 2 2 2 3 3 3 3 3 3
24792024-02-07T18:55:28.653Z03/30 Write block 362 len 8192 data: 5 4
24802024-02-07T18:55:28.653Z04/30 Read block 573 len 32768
24812024-02-07T18:55:28.653Z05/30 Write block 35 len 24576 data: 3 3 3 2 2 3
24822024-02-07T18:55:28.653Z06/30 Read block 291 len 8192
24832024-02-07T18:55:28.653Z07/30 Read block 201 len 24576
24842024-02-07T18:55:28.653Z08/30 Write block 375 len 36864 data: 2 3 3 3 3 3 3 3 3
24852024-02-07T18:55:28.654Z09/30 Read block 436 len 16384
24862024-02-07T18:55:28.654Z10/30 Write block 457 len 12288 data: 2 2 2
24872024-02-07T18:55:28.654Z11/30 Read block 37 len 20480
24882024-02-07T18:55:28.654Z12/30 Read block 546 len 16384
24892024-02-07T18:55:28.654Z13/30 Read block 575 len 4096
24902024-02-07T18:55:28.654Z14/30 Read block 99 len 8192
24912024-02-07T18:55:28.654Z15/30 Write block 84 len 4096 data: 2
24922024-02-07T18:55:28.654Z16/30 Read block 311 len 8192
24932024-02-07T18:55:28.654Z17/30 Write block 271 len 24576 data: 2 2 2 2 2 2
24942024-02-07T18:55:28.654Z18/30 Write block 55 len 16384 data: 4 4 4 4
24952024-02-07T18:55:28.654Z19/30 Write block 366 len 4096 data: 4
24962024-02-07T18:55:28.654Z20/30 Read block 50 len 12288
24972024-02-07T18:55:28.654Z21/30 Read block 105 len 24576
24982024-02-07T18:55:28.654Z22/30 Read block 408 len 24576
24992024-02-07T18:55:28.654Z23/30 Write block 256 len 4096 data: 3
25002024-02-07T18:55:28.654Z24/30 Read block 184 len 40960
25012024-02-07T18:55:28.654Z25/30 Write block 5 len 4096 data: 2
25022024-02-07T18:55:28.655Z26/30 Read block 248 len 20480
25032024-02-07T18:55:28.655Z27/30 Write block 384 len 24576 data: 2 2 2 2 2 3
25042024-02-07T18:55:28.655Z28/30 Write block 20 len 28672 data: 3 3 3 3 3 3 3
25052024-02-07T18:55:28.655Z29/30 Write block 97 len 4096 data: 3
25062024-02-07T18:55:28.655Z30/30 Read block 210 len 20480
25072024-02-07T18:55:28.655Z----------------------------------------------------------------
25082024-02-07T18:55:28.655Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:29
25092024-02-07T18:55:28.655ZStates: Active Active Active
25102024-02-07T18:55:28.655ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
25112024-02-07T18:55:28.655Z 2 Acked 1001 Write 10 Done Done Done false
25122024-02-07T18:55:28.655Z 3 Acked 1002 Write 2 Done Done Done false
25132024-02-07T18:55:28.655Z 4 Acked 1003 Read 8 Done Done Done false
25142024-02-07T18:55:28.655Z 5 Acked 1004 Write 6 Done Done Done false
25152024-02-07T18:55:28.655Z 6 Acked 1005 Read 2 Done Done Done false
25162024-02-07T18:55:28.655Z 7 Acked 1006 Read 6 Done Done Done false
25172024-02-07T18:55:28.655Z 8 Acked 1007 Write 9 Done Done Done false
25182024-02-07T18:55:28.655Z 9 Acked 1008 Read 4 Done Done Sent false
25192024-02-07T18:55:28.655Z 10 Acked 1009 Write 3 Done Done Sent false
25202024-02-07T18:55:28.656Z 11 Acked 1010 Read 5 Done Done Sent false
25212024-02-07T18:55:28.656Z 12 Acked 1011 Read 4 Done Done Sent false
25222024-02-07T18:55:28.656Z 13 Acked 1012 Read 1 Done Done Sent false
25232024-02-07T18:55:28.656Z 14 Acked 1013 Read 2 Done Done Sent false
25242024-02-07T18:55:28.656Z 15 Acked 1014 Write 1 Done Done Sent false
25252024-02-07T18:55:28.656Z 16 Acked 1015 Read 2 Done Done Sent false
25262024-02-07T18:55:28.656Z 17 Acked 1016 Write 6 Done Done Sent false
25272024-02-07T18:55:28.656Z 18 Acked 1017 Write 4 Done Done Sent false
25282024-02-07T18:55:28.656Z 19 Acked 1018 Write 1 Done Done Sent false
25292024-02-07T18:55:28.656Z 20 Acked 1019 Read 3 Done Done Sent false
25302024-02-07T18:55:28.656Z 21 Acked 1020 Read 6 Done Done Sent false
25312024-02-07T18:55:28.656Z 22 Acked 1021 Read 6 Done Done Sent false
25322024-02-07T18:55:28.656Z 23 Acked 1022 Write 1 Done Done Sent false
25332024-02-07T18:55:28.656Z 24 Acked 1023 Read 10 Done Done Sent false
25342024-02-07T18:55:28.656Z 25 Acked 1024 Write 1 Done Done Sent false
25352024-02-07T18:55:28.656Z 26 Acked 1025 Read 5 Done Done Sent false
25362024-02-07T18:55:28.656Z 27 Acked 1026 Write 6 Done Done Sent false
25372024-02-07T18:55:28.656Z 28 Acked 1027 Write 7 Done Done Sent false
25382024-02-07T18:55:28.656Z 29 Acked 1028 Write 1 Done Done Sent false
25392024-02-07T18:55:28.657Z 30 Acked 1029 Read 5 Done Sent Sent false
25402024-02-07T18:55:28.657Z STATES DS:0 DS:1 DS:2 TOTAL
25412024-02-07T18:55:28.657Z New 0 0 0 0
25422024-02-07T18:55:28.657Z Sent 0 1 22 23
25432024-02-07T18:55:28.657Z Done 29 28 7 64
25442024-02-07T18:55:28.657Z Skipped 0 0 0 0
25452024-02-07T18:55:28.669Z Error 0 0 0 0
25462024-02-07T18:55:28.669ZLast Flush: 1000 1000 1000
25472024-02-07T18:55:28.670ZDownstairs last five completed: 1000
25482024-02-07T18:55:28.670ZUpstairs last five completed: 30 29 28 27 26
25492024-02-07T18:55:28.670Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:27.901576672Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1223}
25502024-02-07T18:55:28.671ZWrote out file "/tmp/test_repair_verify.data"
25512024-02-07T18:55:28.671Z
25522024-02-07T18:55:28.671ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25532024-02-07T18:55:28.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25542024-02-07T18:55:28.671Z 0 000-019 17 17 17 34 34 34 T T F <---
25552024-02-07T18:55:28.671Z 1 020-039 14 14 14 30 30 30 T T T
25562024-02-07T18:55:28.671Z 2 040-059 10 10 10 21 21 21 T T T
25572024-02-07T18:55:28.672Z 3 060-079 18 18 18 37 37 37 F F F
25582024-02-07T18:55:28.672Z 4 080-099 18 18 18 37 37 37 T T F <---
25592024-02-07T18:55:28.672Z 5 100-119 20 20 20 42 42 42 F F F
25602024-02-07T18:55:28.672Z 6 120-139 20 20 20 42 42 42 F F F
25612024-02-07T18:55:28.672Z 7 140-159 18 18 18 37 37 37 F F F
25622024-02-07T18:55:28.672Z 8 160-179 20 20 20 41 41 41 F F F
25632024-02-07T18:55:28.672Z 9 180-199 20 20 20 41 41 41 F F F
25642024-02-07T18:55:28.673Z 10 200-219 15 15 15 31 31 31 T T T
25652024-02-07T18:55:28.673Z 11 220-239 18 18 18 37 37 37 T T T
25662024-02-07T18:55:28.673Z 12 240-259 20 20 20 42 42 42 T T F <---
25672024-02-07T18:55:28.673Z 13 260-279 20 20 20 42 42 42 T T F <---
25682024-02-07T18:55:28.673Z 14 280-299 20 20 20 42 42 42 F F F
25692024-02-07T18:55:28.673Z 15 300-319 12 12 12 24 24 24 F F F
25702024-02-07T18:55:28.673Z 16 320-339 21 21 21 43 43 43 F F F
25712024-02-07T18:55:28.674Z 17 340-359 19 19 19 39 39 39 F F F
25722024-02-07T18:55:28.674Z 18 360-379 21 21 21 43 43 43 T T T
25732024-02-07T18:55:28.674Z 19 380-399 20 20 20 42 42 42 T T T
25742024-02-07T18:55:28.674Z 20 400-419 17 17 17 34 34 34 F F F
25752024-02-07T18:55:28.674Z 21 420-439 18 18 18 37 37 37 F F F
25762024-02-07T18:55:28.674Z 22 440-459 18 18 18 36 36 36 T T F <---
25772024-02-07T18:55:28.674Z 23 460-479 16 16 16 33 33 33 F F F
25782024-02-07T18:55:28.675Z 24 480-499 14 14 14 29 29 29 F F F
25792024-02-07T18:55:28.675Z 25 500-519 17 17 17 34 34 34 F F F
25802024-02-07T18:55:28.675Z 26 520-539 12 12 12 27 27 27 F F F
25812024-02-07T18:55:28.675Z 27 540-559 21 21 21 43 43 43 F F F
25822024-02-07T18:55:28.675Z 28 560-579 10 10 10 21 21 21 F F F
25832024-02-07T18:55:28.675Z 29 580-599 18 18 18 37 37 37 F F F
25842024-02-07T18:55:28.675ZMax gen: 21, Max flush: 43
25852024-02-07T18:55:28.681ZError: Difference in extent metadata found!
25862024-02-07T18:55:28.682ZA Difference in extent metadata is expected here
25872024-02-07T18:55:28.682ZOn loop 11
25882024-02-07T18:55:28.682Z
25892024-02-07T18:55:28.682ZVerifying data now
25902024-02-07T18:55:34.245ZNo change after verify
25912024-02-07T18:55:34.246ZLoop: 11 Downstairs dump after verify (and repair):
25922024-02-07T18:55:34.261ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25932024-02-07T18:55:34.261Z 0 000-019 22 22 22 44 44 44 F F F
25942024-02-07T18:55:34.261Z 1 020-039 22 22 22 44 44 44 F F F
25952024-02-07T18:55:34.261Z 2 040-059 22 22 22 44 44 44 F F F
25962024-02-07T18:55:34.261Z 3 060-079 18 18 18 37 37 37 F F F
25972024-02-07T18:55:34.261Z 4 080-099 22 22 22 44 44 44 F F F
25982024-02-07T18:55:34.261Z 5 100-119 20 20 20 42 42 42 F F F
25992024-02-07T18:55:34.261Z 6 120-139 20 20 20 42 42 42 F F F
26002024-02-07T18:55:34.261Z 7 140-159 18 18 18 37 37 37 F F F
26012024-02-07T18:55:34.261Z 8 160-179 20 20 20 41 41 41 F F F
26022024-02-07T18:55:34.268Z 9 180-199 20 20 20 41 41 41 F F F
26032024-02-07T18:55:34.268Z 10 200-219 22 22 22 44 44 44 F F F
26042024-02-07T18:55:34.269Z 11 220-239 22 22 22 44 44 44 F F F
26052024-02-07T18:55:34.269Z 12 240-259 22 22 22 44 44 44 F F F
26062024-02-07T18:55:34.269Z 13 260-279 22 22 22 44 44 44 F F F
26072024-02-07T18:55:34.269Z 14 280-299 20 20 20 42 42 42 F F F
26082024-02-07T18:55:34.269Z 15 300-319 12 12 12 24 24 24 F F F
26092024-02-07T18:55:34.269Z 16 320-339 21 21 21 43 43 43 F F F
26102024-02-07T18:55:34.269Z 17 340-359 19 19 19 39 39 39 F F F
26112024-02-07T18:55:34.269Z 18 360-379 22 22 22 44 44 44 F F F
26122024-02-07T18:55:34.269Z 19 380-399 22 22 22 44 44 44 F F F
26132024-02-07T18:55:34.269Z 20 400-419 17 17 17 34 34 34 F F F
26142024-02-07T18:55:34.269Z 21 420-439 18 18 18 37 37 37 F F F
26152024-02-07T18:55:34.270Z 22 440-459 22 22 22 44 44 44 F F F
26162024-02-07T18:55:34.270Z 23 460-479 16 16 16 33 33 33 F F F
26172024-02-07T18:55:34.270Z 24 480-499 14 14 14 29 29 29 F F F
26182024-02-07T18:55:34.270Z 25 500-519 17 17 17 34 34 34 F F F
26192024-02-07T18:55:34.270Z 26 520-539 12 12 12 27 27 27 F F F
26202024-02-07T18:55:34.270Z 27 540-559 21 21 21 43 43 43 F F F
26212024-02-07T18:55:34.270Z 28 560-579 10 10 10 21 21 21 F F F
26222024-02-07T18:55:34.271Z 29 580-599 18 18 18 37 37 37 F F F
26232024-02-07T18:55:34.271ZMax gen: 22, Max flush: 44
26242024-02-07T18:55:34.271Z
26252024-02-07T18:55:34.271ZBegin loop 12
26262024-02-07T18:55:34.271ZDownstairs to restart: 2
26272024-02-07T18:55:34.284Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.548714784Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231}
26282024-02-07T18:55:34.284Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
26292024-02-07T18:55:34.284Z,"time":"2024-02-07T18:55:33.549033521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231}
26302024-02-07T18:55:34.301Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549068606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231}
26312024-02-07T18:55:34.301Z{"msg":"Crucible 8178cb14-2391-48eb-b58d-751ee1272be4 has session id: 07200021-51b1-447c-aca1-a049a127579e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549109301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26322024-02-07T18:55:34.301Z{"msg":"Upstairs opts: Upstairs UUID: 8178cb14-2391-48eb-b58d-751ee1272be4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549166683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26332024-02-07T18:55:34.302Z{"msg":"Crucible stats registered with UUID: 8178cb14-2391-48eb-b58d-751ee1272be4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549214577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26342024-02-07T18:55:34.302Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549244243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231}
26352024-02-07T18:55:34.302Z{"msg":"8178cb14-2391-48eb-b58d-751ee1272be4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549406961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26362024-02-07T18:55:34.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549458184Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26372024-02-07T18:55:34.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549540132Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26382024-02-07T18:55:34.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549578607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26392024-02-07T18:55:34.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549612923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26402024-02-07T18:55:34.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549645378Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26412024-02-07T18:55:34.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549691442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26422024-02-07T18:55:34.303Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549725367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26432024-02-07T18:55:34.303Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549754974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26442024-02-07T18:55:34.304Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549786709Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26452024-02-07T18:55:34.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549824384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26462024-02-07T18:55:34.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549851641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26472024-02-07T18:55:34.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.549873418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26482024-02-07T18:55:34.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.550487355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26492024-02-07T18:55:34.304Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.550516031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26502024-02-07T18:55:34.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.550531798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26512024-02-07T18:55:34.305Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.550559785Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26522024-02-07T18:55:34.305Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551100972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26532024-02-07T18:55:34.305Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551237173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26542024-02-07T18:55:34.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551364506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26552024-02-07T18:55:34.305Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.55140996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26562024-02-07T18:55:34.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.5517028Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26572024-02-07T18:55:34.305Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551742395Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26582024-02-07T18:55:34.306Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551758852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26592024-02-07T18:55:34.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551828553Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26602024-02-07T18:55:34.306Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551863448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26612024-02-07T18:55:34.306Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.551878376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26622024-02-07T18:55:34.306Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552120334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26632024-02-07T18:55:34.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552568103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26642024-02-07T18:55:34.306Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552607648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26652024-02-07T18:55:34.306Z{"msg":"[0]R flush_numbers[0..12]: [44, 44, 44, 37, 44, 42, 42, 37, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552625825Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26662024-02-07T18:55:34.306Z{"msg":"[0]R generation[0..12]: [22, 22, 22, 18, 22, 20, 20, 18, 20, 20, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552648632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26672024-02-07T18:55:34.306Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.55266383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26682024-02-07T18:55:34.307Z{"msg":"[1]R flush_numbers[0..12]: [44, 44, 44, 37, 44, 42, 42, 37, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552679208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26692024-02-07T18:55:34.307Z{"msg":"[1]R generation[0..12]: [22, 22, 22, 18, 22, 20, 20, 18, 20, 20, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552694306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26702024-02-07T18:55:34.307Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552717163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26712024-02-07T18:55:34.307Z{"msg":"[2]R flush_numbers[0..12]: [44, 44, 44, 37, 44, 42, 42, 37, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552732281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26722024-02-07T18:55:34.307Z{"msg":"[2]R generation[0..12]: [22, 22, 22, 18, 22, 20, 20, 18, 20, 20, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552759287Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26732024-02-07T18:55:34.307Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
26742024-02-07T18:55:34.307Z,"level":30,"time":"2024-02-07T18:55:33.552778185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26752024-02-07T18:55:34.307Z{"msg":"Max found gen is 23","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552791693Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26762024-02-07T18:55:34.307Z{"msg":"Generation requested: 24 >= found:23","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.55281285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26772024-02-07T18:55:34.307Z{"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552827118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26782024-02-07T18:55:34.307Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552847875Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26792024-02-07T18:55:34.307Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552862443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26802024-02-07T18:55:34.307Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552876161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26812024-02-07T18:55:34.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.55288963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26822024-02-07T18:55:34.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552910737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26832024-02-07T18:55:34.308Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552924905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26842024-02-07T18:55:34.308Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552938383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26852024-02-07T18:55:34.308Z{"msg":"8178cb14-2391-48eb-b58d-751ee1272be4 is now active with session: 07200021-51b1-447c-aca1-a049a127579e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552952291Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26862024-02-07T18:55:34.308Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.5529645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231,"session_id":"07200021-51b1-447c-aca1-a049a127579e"}
26872024-02-07T18:55:34.308Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:33.552984527Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231}
26882024-02-07T18:55:34.308ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26892024-02-07T18:55:34.309ZLoading write count information from file "/tmp/test_repair_verify.data"
26902024-02-07T18:55:34.309ZRun Repair workload
26912024-02-07T18:55:34.309Z01/30 Write block 23 len 12288 data: 4 4 4
26922024-02-07T18:55:34.320Z02/30 Write block 335 len 24576 data: 3 3 3 4 6 5
26932024-02-07T18:55:34.320Z03/30 Read block 167 len 40960
26942024-02-07T18:55:34.320Z04/30 Read block 160 len 36864
26952024-02-07T18:55:34.320Z05/30 Read block 211 len 8192
26962024-02-07T18:55:34.320Z06/30 Flush
26972024-02-07T18:55:34.320Z07/30 Write block 7 len 20480 data: 2 2 2 2 2
26982024-02-07T18:55:34.320Z08/30 Read block 551 len 24576
26992024-02-07T18:55:34.320Z09/30 Write block 51 len 16384 data: 5 4 5 5
27002024-02-07T18:55:34.320Z10/30 Write block 442 len 20480 data: 3 3 3 2 2
27012024-02-07T18:55:34.320Z11/30 Write block 242 len 16384 data: 5 6 5 4
27022024-02-07T18:55:34.321Z12/30 Read block 185 len 28672
27032024-02-07T18:55:34.321Z13/30 Write block 90 len 24576 data: 4 4 4 3 4 3
27042024-02-07T18:55:34.321Z14/30 Read block 163 len 8192
27052024-02-07T18:55:34.321Z15/30 Read block 369 len 4096
27062024-02-07T18:55:34.321Z16/30 Read block 363 len 40960
27072024-02-07T18:55:34.321Z17/30 Flush
27082024-02-07T18:55:34.321Z18/30 Write block 507 len 32768 data: 2 2 2 2 2 4 4 4
27092024-02-07T18:55:34.321Z19/30 Read block 161 len 28672
27102024-02-07T18:55:34.321Z20/30 Read block 263 len 4096
27112024-02-07T18:55:34.321Z21/30 Flush
27122024-02-07T18:55:34.321Z22/30 Write block 17 len 32768 data: 4 3 4 4 4 4 5 5
27132024-02-07T18:55:34.321Z23/30 Read block 567 len 20480
27142024-02-07T18:55:34.321Z24/30 Read block 223 len 16384
27152024-02-07T18:55:34.321Z25/30 Read block 333 len 32768
27162024-02-07T18:55:34.321Z26/30 Write block 526 len 8192 data: 2 2
27172024-02-07T18:55:34.321Z27/30 Write block 141 len 16384 data: 3 3 3 3
27182024-02-07T18:55:34.321Z28/30 Read block 41 len 40960
27192024-02-07T18:55:34.321Z29/30 Write block 547 len 12288 data: 2 2 3
27202024-02-07T18:55:34.321Z30/30 Write block 88 len 36864 data: 4 4 5 5 5 4 5 4 3
27212024-02-07T18:55:34.326Z----------------------------------------------------------------
27222024-02-07T18:55:34.326Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
27232024-02-07T18:55:34.326ZStates: Active Active Active
27242024-02-07T18:55:34.326ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27252024-02-07T18:55:34.326Z 1 Acked 1000 Write 3 Done Done Done false
27262024-02-07T18:55:34.326Z 2 Acked 1001 Write 6 Done Done Done false
27272024-02-07T18:55:34.326Z 3 Acked 1002 Read 10 Done Done Done false
27282024-02-07T18:55:34.326Z 4 Acked 1003 Read 9 Done Done Sent false
27292024-02-07T18:55:34.327Z 5 Acked 1004 Read 2 Done Done Sent false
27302024-02-07T18:55:34.327Z 6 Acked 1005 Flush 0 Done Done Sent false
27312024-02-07T18:55:34.327Z 7 Acked 1006 Write 5 Done Done Sent false
27322024-02-07T18:55:34.327Z 8 Acked 1007 Read 6 Done Done Sent false
27332024-02-07T18:55:34.327Z 9 Acked 1008 Write 4 Done Done Sent false
27342024-02-07T18:55:34.327Z 10 Acked 1009 Write 5 Done Done Sent false
27352024-02-07T18:55:34.327Z 11 Acked 1010 Write 4 Done Done Sent false
27362024-02-07T18:55:34.327Z 12 Acked 1011 Read 7 Done Done Sent false
27372024-02-07T18:55:34.327Z 13 Acked 1012 Write 6 Done Done Sent false
27382024-02-07T18:55:34.327Z 14 Acked 1013 Read 2 Done Done Sent false
27392024-02-07T18:55:34.327Z 15 Acked 1014 Read 1 Done Done Sent false
27402024-02-07T18:55:34.327Z 16 Acked 1015 Read 10 Done Done Sent false
27412024-02-07T18:55:34.327Z 17 Acked 1016 Flush 0 Done Done Sent false
27422024-02-07T18:55:34.327Z 18 Acked 1017 Write 8 Done Done Sent false
27432024-02-07T18:55:34.327Z 19 Acked 1018 Read 7 Done Done Sent false
27442024-02-07T18:55:34.327Z 20 Acked 1019 Read 1 Done Done Sent false
27452024-02-07T18:55:34.327Z 21 Acked 1020 Flush 0 Done Done Sent false
27462024-02-07T18:55:34.327Z 22 Acked 1021 Write 8 Done Done Sent false
27472024-02-07T18:55:34.327Z 23 Acked 1022 Read 5 Done Done Sent false
27482024-02-07T18:55:34.327Z 24 Acked 1023 Read 4 Done Done Sent false
27492024-02-07T18:55:34.327Z 25 Acked 1024 Read 8 Done Done Sent false
27502024-02-07T18:55:34.327Z 26 Acked 1025 Write 2 Done Done Sent false
27512024-02-07T18:55:34.327Z 27 Acked 1026 Write 4 Done Done Sent false
27522024-02-07T18:55:34.327Z 28 Acked 1027 Read 10 Done Done Sent false
27532024-02-07T18:55:34.328Z 29 Acked 1028 Write 3 Done Done Sent false
27542024-02-07T18:55:34.328Z 30 Acked 1029 Write 9 Sent Sent Sent false
27552024-02-07T18:55:34.328Z STATES DS:0 DS:1 DS:2 TOTAL
27562024-02-07T18:55:34.328Z New 0 0 0 0
27572024-02-07T18:55:34.328Z Sent 1 1 27 29
27582024-02-07T18:55:34.328Z Done 29 29 3 61
27592024-02-07T18:55:34.328Z Skipped 0 0 0 0
27602024-02-07T18:55:34.328Z Error 0 0 0 0
27612024-02-07T18:55:34.328ZLast Flush: 1020 1020 0
27622024-02-07T18:55:34.328ZDownstairs last five completed:
27632024-02-07T18:55:34.328ZUpstairs last five completed: 30 29 28 27 26
27642024-02-07T18:55:34.328Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:33.583742526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1231}
27652024-02-07T18:55:34.328ZWrote out file "/tmp/test_repair_verify.data"
27662024-02-07T18:55:34.328Z
27672024-02-07T18:55:34.328ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27682024-02-07T18:55:34.344ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27692024-02-07T18:55:34.344Z 0 000-019 24 24 22 46 46 44 T T F <---
27702024-02-07T18:55:34.344Z 1 020-039 24 24 22 45 45 44 T T T <---
27712024-02-07T18:55:34.344Z 2 040-059 24 24 22 46 46 44 F F F <---
27722024-02-07T18:55:34.344Z 3 060-079 18 18 18 37 37 37 F F F
27732024-02-07T18:55:34.344Z 4 080-099 24 24 22 46 46 44 T T F <---
27742024-02-07T18:55:34.344Z 5 100-119 20 20 20 42 42 42 F F F
27752024-02-07T18:55:34.344Z 6 120-139 20 20 20 42 42 42 F F F
27762024-02-07T18:55:34.344Z 7 140-159 18 18 18 37 37 37 T T F <---
27772024-02-07T18:55:34.344Z 8 160-179 20 20 20 41 41 41 F F F
27782024-02-07T18:55:34.344Z 9 180-199 20 20 20 41 41 41 F F F
27792024-02-07T18:55:34.344Z 10 200-219 22 22 22 44 44 44 F F F
27802024-02-07T18:55:34.345Z 11 220-239 22 22 22 44 44 44 F F F
27812024-02-07T18:55:34.345Z 12 240-259 24 24 22 46 46 44 F F F <---
27822024-02-07T18:55:34.345Z 13 260-279 22 22 22 44 44 44 F F F
27832024-02-07T18:55:34.345Z 14 280-299 20 20 20 42 42 42 F F F
27842024-02-07T18:55:34.345Z 15 300-319 12 12 12 24 24 24 F F F
27852024-02-07T18:55:34.345Z 16 320-339 24 24 21 45 45 43 F F T <---
27862024-02-07T18:55:34.345Z 17 340-359 24 24 19 45 45 39 F F T <---
27872024-02-07T18:55:34.345Z 18 360-379 22 22 22 44 44 44 F F F
27882024-02-07T18:55:34.345Z 19 380-399 22 22 22 44 44 44 F F F
27892024-02-07T18:55:34.351Z 20 400-419 17 17 17 34 34 34 F F F
27902024-02-07T18:55:34.351Z 21 420-439 18 18 18 37 37 37 F F F
27912024-02-07T18:55:34.351Z 22 440-459 24 24 22 46 46 44 F F F <---
27922024-02-07T18:55:34.351Z 23 460-479 16 16 16 33 33 33 F F F
27932024-02-07T18:55:34.351Z 24 480-499 14 14 14 29 29 29 F F F
27942024-02-07T18:55:34.351Z 25 500-519 24 24 17 47 47 34 F F F <---
27952024-02-07T18:55:34.351Z 26 520-539 12 12 12 27 27 27 T T F <---
27962024-02-07T18:55:34.351Z 27 540-559 21 21 21 43 43 43 T T F <---
27972024-02-07T18:55:34.351Z 28 560-579 10 10 10 21 21 21 F F F
27982024-02-07T18:55:34.351Z 29 580-599 18 18 18 37 37 37 F F F
27992024-02-07T18:55:34.351ZMax gen: 24, Max flush: 47
28002024-02-07T18:55:34.351ZError: Difference in extent metadata found!
28012024-02-07T18:55:34.351ZA Difference in extent metadata is expected here
28022024-02-07T18:55:34.351ZOn loop 12
28032024-02-07T18:55:34.351Z
28042024-02-07T18:55:34.359ZVerifying data now
28052024-02-07T18:55:39.969ZNo change after verify
28062024-02-07T18:55:39.969ZLoop: 12 Downstairs dump after verify (and repair):
28072024-02-07T18:55:39.983ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
28082024-02-07T18:55:39.983Z 0 000-019 25 25 25 48 48 48 F F F
28092024-02-07T18:55:39.983Z 1 020-039 25 25 25 48 48 48 F F F
28102024-02-07T18:55:39.983Z 2 040-059 24 24 24 46 46 46 F F F
28112024-02-07T18:55:39.983Z 3 060-079 18 18 18 37 37 37 F F F
28122024-02-07T18:55:39.983Z 4 080-099 25 25 25 48 48 48 F F F
28132024-02-07T18:55:39.983Z 5 100-119 20 20 20 42 42 42 F F F
28142024-02-07T18:55:39.983Z 6 120-139 20 20 20 42 42 42 F F F
28152024-02-07T18:55:39.983Z 7 140-159 25 25 25 48 48 48 F F F
28162024-02-07T18:55:39.983Z 8 160-179 20 20 20 41 41 41 F F F
28172024-02-07T18:55:39.990Z 9 180-199 20 20 20 41 41 41 F F F
28182024-02-07T18:55:39.990Z 10 200-219 22 22 22 44 44 44 F F F
28192024-02-07T18:55:39.991Z 11 220-239 22 22 22 44 44 44 F F F
28202024-02-07T18:55:39.991Z 12 240-259 24 24 24 46 46 46 F F F
28212024-02-07T18:55:39.991Z 13 260-279 22 22 22 44 44 44 F F F
28222024-02-07T18:55:39.991Z 14 280-299 20 20 20 42 42 42 F F F
28232024-02-07T18:55:39.991Z 15 300-319 12 12 12 24 24 24 F F F
28242024-02-07T18:55:39.991Z 16 320-339 24 24 24 45 45 45 F F F
28252024-02-07T18:55:39.991Z 17 340-359 24 24 24 45 45 45 F F F
28262024-02-07T18:55:39.991Z 18 360-379 22 22 22 44 44 44 F F F
28272024-02-07T18:55:39.991Z 19 380-399 22 22 22 44 44 44 F F F
28282024-02-07T18:55:39.991Z 20 400-419 17 17 17 34 34 34 F F F
28292024-02-07T18:55:39.991Z 21 420-439 18 18 18 37 37 37 F F F
28302024-02-07T18:55:39.991Z 22 440-459 24 24 24 46 46 46 F F F
28312024-02-07T18:55:39.991Z 23 460-479 16 16 16 33 33 33 F F F
28322024-02-07T18:55:39.991Z 24 480-499 14 14 14 29 29 29 F F F
28332024-02-07T18:55:39.991Z 25 500-519 24 24 24 47 47 47 F F F
28342024-02-07T18:55:39.991Z 26 520-539 25 25 25 48 48 48 F F F
28352024-02-07T18:55:39.991Z 27 540-559 25 25 25 48 48 48 F F F
28362024-02-07T18:55:39.992Z 28 560-579 10 10 10 21 21 21 F F F
28372024-02-07T18:55:39.992Z 29 580-599 18 18 18 37 37 37 F F F
28382024-02-07T18:55:39.992ZMax gen: 25, Max flush: 48
28392024-02-07T18:55:39.992Z
28402024-02-07T18:55:39.992ZBegin loop 13
28412024-02-07T18:55:39.992ZDownstairs to restart: 2
28422024-02-07T18:55:40.006Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.268897976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239}
28432024-02-07T18:55:40.006Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269172538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239}
28442024-02-07T18:55:40.023Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269209193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":Crucible runtime is spawned
28452024-02-07T18:55:40.023Z1239}
28462024-02-07T18:55:40.023Z{"msg":"Crucible 89bf6995-e6d0-40fc-bbab-1084c9f318fe has session id: f6b6bc5c-0de7-4d0b-83c7-f9b39f995153","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269297191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28472024-02-07T18:55:40.023Z{"msg":"Upstairs opts: Upstairs UUID: 89bf6995-e6d0-40fc-bbab-1084c9f318fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269371991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28482024-02-07T18:55:40.023Z{"msg":"Crucible stats registered with UUID: 89bf6995-e6d0-40fc-bbab-1084c9f318fe","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269435893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28492024-02-07T18:55:40.023Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269473987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239}
28502024-02-07T18:55:40.023Z{"msg":"89bf6995-e6d0-40fc-bbab-1084c9f318fe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269513742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28512024-02-07T18:55:40.023Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269555396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28522024-02-07T18:55:40.023Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269632606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28532024-02-07T18:55:40.023Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269685319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28542024-02-07T18:55:40.023Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269735012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28552024-02-07T18:55:40.024Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269765438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28562024-02-07T18:55:40.024Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269793204Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28572024-02-07T18:55:40.024Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.26982397Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28582024-02-07T18:55:40.024Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269875163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28592024-02-07T18:55:40.024Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269926156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28602024-02-07T18:55:40.024Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269949093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28612024-02-07T18:55:40.024Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.26997635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28622024-02-07T18:55:40.024Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.269994147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28632024-02-07T18:55:40.024Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.270458225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28642024-02-07T18:55:40.024Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.270495639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28652024-02-07T18:55:40.024Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.270511947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28662024-02-07T18:55:40.024Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.270529055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28672024-02-07T18:55:40.024Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27115458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28682024-02-07T18:55:40.024Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271207293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28692024-02-07T18:55:40.024Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271631396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28702024-02-07T18:55:40.024Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27167547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28712024-02-07T18:55:40.025Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271810971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28722024-02-07T18:55:40.025Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271843827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28732024-02-07T18:55:40.025Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271859535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28742024-02-07T18:55:40.025Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271872443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28752024-02-07T18:55:40.025Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27189362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28762024-02-07T18:55:40.025Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.271907688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28772024-02-07T18:55:40.025Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27219023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28782024-02-07T18:55:40.025Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272647058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28792024-02-07T18:55:40.025Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272688973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28802024-02-07T18:55:40.025Z{"msg":"[0]R flush_numbers[0..12]: [48, 48, 46, 37, 48, 42, 42, 48, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27270687Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28812024-02-07T18:55:40.025Z{"msg":"[0]R generation[0..12]: [25, 25, 24, 18, 25, 20, 20, 25, 20, 20, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272741036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28822024-02-07T18:55:40.025Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272775141Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28832024-02-07T18:55:40.025Z{"msg":"[1]R flush_numbers[0..12]: [48, 48, 46, 37, 48, 42, 42, 48, 41, 41, 44, 44]","v":0,"Wait for a query_work_queue command to finish before sending IO
28842024-02-07T18:55:40.025Zname":"crucible","level":30,"time":"2024-02-07T18:55:39.272810866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28852024-02-07T18:55:40.025Z{"msg":"[1]R generation[0..12]: [25, 25, 24, 18, 25, 20, 20, 25, 20, 20, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272831164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28862024-02-07T18:55:40.025Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272862429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28872024-02-07T18:55:40.026Z{"msg":"[2]R flush_numbers[0..12]: [48, 48, 46, 37, 48, 42, 42, 48, 41, 41, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272881717Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28882024-02-07T18:55:40.026Z{"msg":"[2]R generation[0..12]: [25, 25, 24, 18, 25, 20, 20, 25, 20, 20, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272911093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28892024-02-07T18:55:40.026Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27292919Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28902024-02-07T18:55:40.026Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272959746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28912024-02-07T18:55:40.026Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.272977064Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28922024-02-07T18:55:40.026Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273018518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28932024-02-07T18:55:40.026Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273035396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28942024-02-07T18:55:40.026Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273049924Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28952024-02-07T18:55:40.026Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27308076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28962024-02-07T18:55:40.026Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273112386Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28972024-02-07T18:55:40.026Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273143541Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28982024-02-07T18:55:40.026Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273158719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
28992024-02-07T18:55:40.026Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273179506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
29002024-02-07T18:55:40.026Zf6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
29012024-02-07T18:55:40.026Z{"msg":"89bf6995-e6d0-40fc-bbab-1084c9f318fe is now active with session: f6b6bc5c-0de7-4d0b-83c7-f9b39f995153","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273210552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
29022024-02-07T18:55:40.026Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.27322444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239,"session_id":"f6b6bc5c-0de7-4d0b-83c7-f9b39f995153"}
29032024-02-07T18:55:40.027Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:39.273255526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239}
29042024-02-07T18:55:40.027ZLoading write count information from file "/tmp/test_repair_verify.data"
29052024-02-07T18:55:40.027ZRun Repair workload
29062024-02-07T18:55:40.027Z01/30 Write block 256 len 40960 data: 4 3 3 3 3 3 3 3 3 2
29072024-02-07T18:55:40.027Z02/30 Read block 62 len 16384
29082024-02-07T18:55:40.027Z03/30 Write block 51 len 28672 data: 6 5 6 6 5 5 5
29092024-02-07T18:55:40.035Z04/30 Flush
29102024-02-07T18:55:40.035Z05/30 Write block 122 len 4096 data: 3
29112024-02-07T18:55:40.035Z06/30 Read block 570 len 12288
29122024-02-07T18:55:40.035Z07/30 Read block 491 len 20480
29132024-02-07T18:55:40.035Z08/30 Write block 94 len 12288 data: 6 5 4
29142024-02-07T18:55:40.035Z09/30 Write block 488 len 12288 data: 3 3 4
29152024-02-07T18:55:40.035Z10/30 Read block 241 len 16384
29162024-02-07T18:55:40.035Z11/30 Read block 396 len 4096
29172024-02-07T18:55:40.035Z12/30 Read block 480 len 28672
29182024-02-07T18:55:40.035Z13/30 Write block 148 len 32768 data: 4 4 3 2 2 2 2 2
29192024-02-07T18:55:40.035Z14/30 Write block 168 len 24576 data: 7 7 6 6 5 5
29202024-02-07T18:55:40.035Z15/30 Write block 233 len 12288 data: 4 6 7
29212024-02-07T18:55:40.036Z16/30 Read block 105 len 20480
29222024-02-07T18:55:40.036Z17/30 Write block 284 len 28672 data: 3 3 4 4 4 4 5
29232024-02-07T18:55:40.036Z18/30 Write block 109 len 12288 data: 3 2 2
29242024-02-07T18:55:40.036Z19/30 Read block 133 len 12288
29252024-02-07T18:55:40.036Z20/30 Read block 461 len 12288
29262024-02-07T18:55:40.036Z21/30 Flush
29272024-02-07T18:55:40.036Z22/30 Write block 223 len 12288 data: 4 4 2
29282024-02-07T18:55:40.036Z23/30 Write block 518 len 12288 data: 3 3 3
29292024-02-07T18:55:40.036Z24/30 Read block 534 len 24576
29302024-02-07T18:55:40.036Z25/30 Write block 172 len 16384 data: 6 6 4 4
29312024-02-07T18:55:40.036Z26/30 Write block 573 len 20480 data: 2 2 2 2 2
29322024-02-07T18:55:40.036Z27/30 Write block 381 len 40960 data: 4 4 4 3 3 3 3 3 4 3
29332024-02-07T18:55:40.042Z28/30 Write block 442 len 28672 data: 4 4 4 3 3 2 2
29342024-02-07T18:55:40.042Z29/30 Write block 477 len 32768 data: 3 3 2 2 2 2 2 2
29352024-02-07T18:55:40.043Z30/30 Write block 290 len 16384 data: 6 3 2 2
29362024-02-07T18:55:40.043Z----------------------------------------------------------------
29372024-02-07T18:55:40.043Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:26
29382024-02-07T18:55:40.043ZStates: Active Active Active
29392024-02-07T18:55:40.043ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29402024-02-07T18:55:40.043Z 5 Acked 1004 Write 1 Done Done Done false
29412024-02-07T18:55:40.043Z 6 Acked 1005 Read 3 Done Done Done false
29422024-02-07T18:55:40.043Z 7 Acked 1006 Read 5 Done Done Done false
29432024-02-07T18:55:40.043Z 8 Acked 1007 Write 3 Done Done Done false
29442024-02-07T18:55:40.043Z 9 Acked 1008 Write 3 Done Done Done false
29452024-02-07T18:55:40.043Z 10 Acked 1009 Read 4 Done Done Done false
29462024-02-07T18:55:40.044Z 11 Acked 1010 Read 1 Done Done Sent false
29472024-02-07T18:55:40.044Z 12 Acked 1011 Read 7 Done Done Sent false
29482024-02-07T18:55:40.044Z 13 Acked 1012 Write 8 Done Done Sent false
29492024-02-07T18:55:40.044Z 14 Acked 1013 Write 6 Done Done Sent false
29502024-02-07T18:55:40.044Z 15 Acked 1014 Write 3 Done Done Sent false
29512024-02-07T18:55:40.044Z 16 Acked 1015 Read 5 Done Done Sent false
29522024-02-07T18:55:40.044Z 17 Acked 1016 Write 7 Done Done Sent false
29532024-02-07T18:55:40.044Z 18 Acked 1017 Write 3 Done Done Sent false
29542024-02-07T18:55:40.044Z 19 Acked 1018 Read 3 Done Done Sent false
29552024-02-07T18:55:40.044Z 20 Acked 1019 Read 3 Done Done Sent false
29562024-02-07T18:55:40.044Z 21 Acked 1020 Flush 0 Done Done Sent false
29572024-02-07T18:55:40.044Z 22 Acked 1021 Write 3 Done Done Sent false
29582024-02-07T18:55:40.044Z 23 Acked 1022 Write 3 Done Done Sent false
29592024-02-07T18:55:40.045Z 24 Acked 1023 Read 6 Done Done Sent false
29602024-02-07T18:55:40.045Z 25 Acked 1024 Write 4 Done Done Sent false
29612024-02-07T18:55:40.045Z 26 Acked 1025 Write 5 Done Done Sent false
29622024-02-07T18:55:40.045Z 27 Acked 1026 Write 10 Done Done Sent false
29632024-02-07T18:55:40.045Z 28 Acked 1027 Write 7 Done Done Sent false
29642024-02-07T18:55:40.045Z 29 Acked 1028 Write 8 Sent Sent Sent false
29652024-02-07T18:55:40.045Z 30 Acked 1029 Write 4 Sent Sent Sent false
29662024-02-07T18:55:40.045Z STATES DS:0 DS:1 DS:2 TOTAL
29672024-02-07T18:55:40.045Z New 0 0 0 0
29682024-02-07T18:55:40.045Z Sent 2 2 20 24
29692024-02-07T18:55:40.045Z Done 24 24 6 54
29702024-02-07T18:55:40.045Z Skipped 0 0 0 0
29712024-02-07T18:55:40.045Z Error 0 0 0 0
29722024-02-07T18:55:40.046ZLast Flush: 1020 1020 1003
29732024-02-07T18:55:40.046ZDownstairs last five completed: 1003 1002 1001 1000
29742024-02-07T18:55:40.046ZUpstairs last five completed: 30 29 28 27 26
29752024-02-07T18:55:40.046Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:39.301290324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1239}
29762024-02-07T18:55:40.046ZWrote out file "/tmp/test_repair_verify.data"
29772024-02-07T18:55:40.046Z
29782024-02-07T18:55:40.054ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29792024-02-07T18:55:40.062ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29802024-02-07T18:55:40.062Z 0 000-019 25 25 25 48 48 48 F F F
29812024-02-07T18:55:40.063Z 1 020-039 25 25 25 48 48 48 F F F
29822024-02-07T18:55:40.063Z 2 040-059 26 26 26 49 49 49 F F F
29832024-02-07T18:55:40.063Z 3 060-079 18 18 18 37 37 37 F F F
29842024-02-07T18:55:40.063Z 4 080-099 26 26 25 50 50 48 F F T <---
29852024-02-07T18:55:40.063Z 5 100-119 26 26 20 50 50 42 F F F <---
29862024-02-07T18:55:40.063Z 6 120-139 26 26 20 50 50 42 F F T <---
29872024-02-07T18:55:40.063Z 7 140-159 26 26 25 50 50 48 F F F <---
29882024-02-07T18:55:40.063Z 8 160-179 26 26 20 50 50 41 T T F <---
29892024-02-07T18:55:40.063Z 9 180-199 20 20 20 41 41 41 F F F
29902024-02-07T18:55:40.063Z 10 200-219 22 22 22 44 44 44 F F F
29912024-02-07T18:55:40.076Z 11 220-239 26 26 22 50 50 44 T T F <---
29922024-02-07T18:55:40.076Z 12 240-259 26 26 26 49 49 49 F F F
29932024-02-07T18:55:40.076Z 13 260-279 26 26 26 49 49 49 F F F
29942024-02-07T18:55:40.076Z 14 280-299 26 26 20 50 50 42 T T F <---
29952024-02-07T18:55:40.076Z 15 300-319 12 12 12 24 24 24 F F F
29962024-02-07T18:55:40.076Z 16 320-339 24 24 24 45 45 45 F F F
29972024-02-07T18:55:40.076Z 17 340-359 24 24 24 45 45 45 F F F
29982024-02-07T18:55:40.076Z 18 360-379 22 22 22 44 44 44 F F F
29992024-02-07T18:55:40.076Z 19 380-399 22 22 22 44 44 44 T T F <---
30002024-02-07T18:55:40.077Z 20 400-419 17 17 17 34 34 34 F F F
30012024-02-07T18:55:40.077Z 21 420-439 18 18 18 37 37 37 F F F
30022024-02-07T18:55:40.077Z 22 440-459 24 24 24 46 46 46 T T F <---
30032024-02-07T18:55:40.077Z 23 460-479 16 16 16 33 33 33 T T F <---
30042024-02-07T18:55:40.077Z 24 480-499 26 26 14 50 50 29 T T T <---
30052024-02-07T18:55:40.077Z 25 500-519 24 24 24 47 47 47 T T F <---
30062024-02-07T18:55:40.077Z 26 520-539 25 25 25 48 48 48 T T F <---
30072024-02-07T18:55:40.077Z 27 540-559 25 25 25 48 48 48 F F F
30082024-02-07T18:55:40.077Z 28 560-579 10 10 10 21 21 21 T T F <---
30092024-02-07T18:55:40.077Z 29 580-599 18 18 18 37 37 37 F F F
30102024-02-07T18:55:40.077ZMax gen: 26, Max flush: 50
30112024-02-07T18:55:40.077ZError: Difference in extent metadata found!
30122024-02-07T18:55:40.077ZA Difference in extent metadata is expected here
30132024-02-07T18:55:40.078ZOn loop 13
30142024-02-07T18:55:40.078Z
30152024-02-07T18:55:40.097ZVerifying data now
30162024-02-07T18:55:45.776ZNo change after verify
30172024-02-07T18:55:45.777ZLoop: 13 Downstairs dump after verify (and repair):
30182024-02-07T18:55:45.811ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30192024-02-07T18:55:45.812Z 0 000-019 25 25 25 48 48 48 F F F
30202024-02-07T18:55:45.812Z 1 020-039 25 25 25 48 48 48 F F F
30212024-02-07T18:55:45.812Z 2 040-059 26 26 26 49 49 49 F F F
30222024-02-07T18:55:45.812Z 3 060-079 18 18 18 37 37 37 F F F
30232024-02-07T18:55:45.812Z 4 080-099 26 26 26 50 50 50 F F F
30242024-02-07T18:55:45.812Z 5 100-119 26 26 26 50 50 50 F F F
30252024-02-07T18:55:45.813Z 6 120-139 26 26 26 50 50 50 F F F
30262024-02-07T18:55:46.243Z 7 140-159 26 26 26 50 50 50 F F F
30272024-02-07T18:55:46.244Z 8 160-179 27 27 27 51 51 51 F F F
30282024-02-07T18:55:46.244Z 9 180-199 20 20 20 41 41 41 F F F
30292024-02-07T18:55:46.244Z 10 200-219 22 22 22 44 44 44 F F F
30302024-02-07T18:55:46.244Z 11 220-239 27 27 27 51 51 51 F F F
30312024-02-07T18:55:46.244Z 12 240-259 26 26 26 49 49 49 F F F
30322024-02-07T18:55:46.244Z 13 260-279 26 26 26 49 49 49 F F F
30332024-02-07T18:55:46.244Z 14 280-299 27 27 27 51 51 51 F F F
30342024-02-07T18:55:46.244Z 15 300-319 12 12 12 24 24 24 F F F
30352024-02-07T18:55:46.244Z 16 320-339 24 24 24 45 45 45 F F F
30362024-02-07T18:55:46.244Z 17 340-359 24 24 24 45 45 45 F F F
30372024-02-07T18:55:46.244Z 18 360-379 22 22 22 44 44 44 F F F
30382024-02-07T18:55:46.244Z 19 380-399 27 27 27 51 51 51 F F F
30392024-02-07T18:55:46.245Z 20 400-419 17 17 17 34 34 34 F F F
30402024-02-07T18:55:46.245Z 21 420-439 18 18 18 37 37 37 F F F
30412024-02-07T18:55:46.245Z 22 440-459 27 27 27 51 51 51 F F F
30422024-02-07T18:55:46.245Z 23 460-479 27 27 27 51 51 51 F F F
30432024-02-07T18:55:46.245Z 24 480-499 27 27 27 51 51 51 F F F
30442024-02-07T18:55:46.245Z 25 500-519 27 27 27 51 51 51 F F F
30452024-02-07T18:55:46.245Z 26 520-539 27 27 27 51 51 51 F F F
30462024-02-07T18:55:46.245Z 27 540-559 25 25 25 48 48 48 F F F
30472024-02-07T18:55:46.245Z 28 560-579 27 27 27 51 51 51 F F F
30482024-02-07T18:55:46.245Z 29 580-599 18 18 18 37 37 37 F F F
30492024-02-07T18:55:46.245ZMax gen: 27, Max flush: 51
30502024-02-07T18:55:46.245Z
30512024-02-07T18:55:46.245ZBegin loop 14
30522024-02-07T18:55:46.246ZDownstairs to restart: 0
30532024-02-07T18:55:46.246Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07200981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247}
30542024-02-07T18:55:46.246Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
30552024-02-07T18:55:46.246Z,"time":"2024-02-07T18:55:45.072397517Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247}
30562024-02-07T18:55:46.246Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072443091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247}
30572024-02-07T18:55:46.246Z{"msg":"Crucible 461f9361-f02e-40c2-98e2-196d2b8b6b04 has session id: 9e12740f-c000-46fb-9253-543f2fca8a11","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072467168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30582024-02-07T18:55:46.246Z{"msg":"Upstairs opts: Upstairs UUID: 461f9361-f02e-40c2-98e2-196d2b8b6b04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072505853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30592024-02-07T18:55:46.246Z{"msg":"Crucible stats registered with UUID: 461f9361-f02e-40c2-98e2-196d2b8b6b04","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07252144Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30602024-02-07T18:55:46.246Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072534109Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247}
30612024-02-07T18:55:46.246Z{"msg":"461f9361-f02e-40c2-98e2-196d2b8b6b04 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072661472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30622024-02-07T18:55:46.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072697507Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30632024-02-07T18:55:46.246Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072717354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30642024-02-07T18:55:46.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072743111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30652024-02-07T18:55:46.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072772457Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30662024-02-07T18:55:46.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072811991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30672024-02-07T18:55:46.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072839648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30682024-02-07T18:55:46.247Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072882492Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30692024-02-07T18:55:46.247Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072931975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30702024-02-07T18:55:46.247Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.072959891Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30712024-02-07T18:55:46.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073017293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30722024-02-07T18:55:46.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073065047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30732024-02-07T18:55:46.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073093783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30742024-02-07T18:55:46.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073717379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30752024-02-07T18:55:46.247Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073754464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30762024-02-07T18:55:46.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073769982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30772024-02-07T18:55:46.248Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.073826454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30782024-02-07T18:55:46.248Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.074387918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30792024-02-07T18:55:46.248Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.074436592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30802024-02-07T18:55:46.248Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07489077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30812024-02-07T18:55:46.248Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.074943043Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30822024-02-07T18:55:46.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075023542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30832024-02-07T18:55:46.248Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075072806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30842024-02-07T18:55:46.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075102651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30852024-02-07T18:55:46.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075140466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30862024-02-07T18:55:46.248Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075158194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30872024-02-07T18:55:46.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075181601Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30882024-02-07T18:55:46.248Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075542592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30892024-02-07T18:55:46.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.075982383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30902024-02-07T18:55:46.249Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076019507Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30912024-02-07T18:55:46.249Z{"msg":"[0]R flush_numbers[0..12]: [48, 48, 49, 37, 50, 50, 50, 50, 51, 41, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076037515Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30922024-02-07T18:55:46.249Z{"msg":"[0]R generation[0..12]: [25, 25, 26, 18, 26, 26, 26, 26, 27, 20, 22, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076052953Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30932024-02-07T18:55:46.249Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07607656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30942024-02-07T18:55:46.249Z{"msg":"[1]R flush_numbers[0..12]: [48, 48, 49, 37, 50, 50, 50, 50, 51, 41, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076092208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30952024-02-07T18:55:46.249Z{"msg":"[1]R generation[0..12]: [25, 25, 26, 18, 26, 26, 26, 26, 27, 20, 22, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076113165Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30962024-02-07T18:55:46.249Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076128073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30972024-02-07T18:55:46.249Z{"msg":"[2]R flush_numbers[0..12]: [48, 48, 49, 37, 50, 50, 50, 50, 51, 41, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07614868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
30982024-02-07T18:55:46.249Z{"msg":"[2]R generation[0..12]: [25, 25, 26, 18, 26, 26, 26, 26, 27, 20, 22, 27]","v":0,"name":Wait for a query_work_queue command to finish before sending IO
30992024-02-07T18:55:46.249Z"crucible","level":30,"time":"2024-02-07T18:55:45.076167238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31002024-02-07T18:55:46.249Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076188644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31012024-02-07T18:55:46.249Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076213981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31022024-02-07T18:55:46.250Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076228539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31032024-02-07T18:55:46.250Z{"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076241728Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31042024-02-07T18:55:46.250Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076262694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31052024-02-07T18:55:46.250Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076277463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31062024-02-07T18:55:46.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07630058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31072024-02-07T18:55:46.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076317587Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31082024-02-07T18:55:46.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076331885Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31092024-02-07T18:55:46.250Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076353922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31102024-02-07T18:55:46.250Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.07636849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31112024-02-07T18:55:46.250Z{"msg":"461f9361-f02e-40c2-98e2-196d2b8b6b04 is now active with session: 9e12740f-c000-46fb-9253-543f2fca8a11","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076388437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31122024-02-07T18:55:46.250Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076401086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247,"session_id":"9e12740f-c000-46fb-9253-543f2fca8a11"}
31132024-02-07T18:55:46.250Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:45.076420223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247}
31142024-02-07T18:55:46.250ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31152024-02-07T18:55:46.251ZLoading write count information from file "/tmp/test_repair_verify.data"
31162024-02-07T18:55:46.251ZRun Repair workload
31172024-02-07T18:55:46.251Z01/30 Write block 355 len 4096 data: 3
31182024-02-07T18:55:46.251Z02/30 Read block 435 len 4096
31192024-02-07T18:55:46.251Z03/30 Read block 112 len 28672
31202024-02-07T18:55:46.251Z04/30 Read block 526 len 12288
31212024-02-07T18:55:46.251Z05/30 Write block 449 len 28672 data: 2 2 2 2 3 2 2
31222024-02-07T18:55:46.251Z06/30 Read block 108 len 8192
31232024-02-07T18:55:46.251Z07/30 Flush
31242024-02-07T18:55:46.251Z08/30 Write block 186 len 12288 data: 2 2 2
31252024-02-07T18:55:46.251Z09/30 Write block 435 len 36864 data: 2 3 4 3 3 4 3 5 5
31262024-02-07T18:55:46.258Z10/30 Read block 396 len 12288
31272024-02-07T18:55:46.258Z11/30 Write block 282 len 32768 data: 3 3 4 4 5 5 5 5
31282024-02-07T18:55:46.258Z12/30 Write block 104 len 8192 data: 2 2
31292024-02-07T18:55:46.258Z13/30 Read block 113 len 28672
31302024-02-07T18:55:46.258Z14/30 Read block 140 len 32768
31312024-02-07T18:55:46.258Z15/30 Read block 315 len 32768
31322024-02-07T18:55:46.258Z16/30 Write block 564 len 20480 data: 3 3 2 2 2
31332024-02-07T18:55:46.259Z17/30 Write block 346 len 20480 data: 5 3 3 3 4
31342024-02-07T18:55:46.259Z18/30 Write block 417 len 20480 data: 4 4 4 4 4
31352024-02-07T18:55:46.259Z19/30 Write block 532 len 12288 data: 2 2 2
31362024-02-07T18:55:46.259Z20/30 Write block 104 len 40960 data: 3 3 3 3 3 4 3 3 2 2
31372024-02-07T18:55:46.259Z21/30 Write block 439 len 32768 data: 4 5 4 6 6 5 4 4
31382024-02-07T18:55:46.259Z22/30 Flush
31392024-02-07T18:55:46.259Z23/30 Write block 28 len 36864 data: 2 3 3 3 3 3 3 4 4
31402024-02-07T18:55:46.259Z24/30 Read block 195 len 32768
31412024-02-07T18:55:46.259Z25/30 Read block 48 len 24576
31422024-02-07T18:55:46.259Z26/30 Read block 377 len 12288
31432024-02-07T18:55:46.259Z27/30 Write block 304 len 36864 data: 2 2 2 2 2 2 2 2 2
31442024-02-07T18:55:46.259Z28/30 Read block 35 len 24576
31452024-02-07T18:55:46.259Z29/30 Read block 406 len 12288
31462024-02-07T18:55:46.259Z30/30 Read block 189 len 36864
31472024-02-07T18:55:46.259Z----------------------------------------------------------------
31482024-02-07T18:55:46.260Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31492024-02-07T18:55:46.260ZStates: Active Active Active
31502024-02-07T18:55:46.260ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31512024-02-07T18:55:46.260Z 1 Acked 1000 Write 1 Done Done Done false
31522024-02-07T18:55:46.260Z 2 Acked 1001 Read 1 Sent Done Done false
31532024-02-07T18:55:46.260Z 3 Acked 1002 Read 7 Sent Done Done false
31542024-02-07T18:55:46.260Z 4 Acked 1003 Read 3 Sent Done Done false
31552024-02-07T18:55:46.260Z 5 Acked 1004 Write 7 Sent Done Done false
31562024-02-07T18:55:46.260Z 6 Acked 1005 Read 2 Sent Done Done false
31572024-02-07T18:55:46.260Z 7 Acked 1006 Flush 0 Sent Done Done false
31582024-02-07T18:55:46.260Z 8 Acked 1007 Write 3 Sent Done Done false
31592024-02-07T18:55:46.260Z 9 Acked 1008 Write 9 Sent Done Done false
31602024-02-07T18:55:46.260Z 10 Acked 1009 Read 3 Sent Done Done false
31612024-02-07T18:55:46.260Z 11 Acked 1010 Write 8 Sent Done Done false
31622024-02-07T18:55:46.260Z 12 Acked 1011 Write 2 Sent Done Done false
31632024-02-07T18:55:46.260Z 13 Acked 1012 Read 7 Sent Done Done false
31642024-02-07T18:55:46.260Z 14 Acked 1013 Read 8 Sent Done Done false
31652024-02-07T18:55:46.260Z 15 Acked 1014 Read 8 Sent Done Done false
31662024-02-07T18:55:46.260Z 16 Acked 1015 Write 5 Sent Done Done false
31672024-02-07T18:55:46.261Z 17 Acked 1016 Write 5 Sent Done Done false
31682024-02-07T18:55:46.261Z 18 Acked 1017 Write 5 Sent Done Done false
31692024-02-07T18:55:46.261Z 19 Acked 1018 Write 3 Sent Done Done false
31702024-02-07T18:55:46.261Z 20 Acked 1019 Write 10 Sent Done Done false
31712024-02-07T18:55:46.261Z 21 Acked 1020 Write 8 Sent Done Done false
31722024-02-07T18:55:46.261Z 22 Acked 1021 Flush 0 Sent Done Done false
31732024-02-07T18:55:46.261Z 23 Acked 1022 Write 9 Sent Done Done false
31742024-02-07T18:55:46.261Z 24 Acked 1023 Read 8 Sent Done Done false
31752024-02-07T18:55:46.261Z 25 Acked 1024 Read 6 Sent Done Done false
31762024-02-07T18:55:46.261Z 26 Acked 1025 Read 3 Sent Done Done false
31772024-02-07T18:55:46.261Z 27 Acked 1026 Write 9 Sent Done Done false
31782024-02-07T18:55:46.261Z 28 Acked 1027 Read 6 Sent Done Done false
31792024-02-07T18:55:46.261Z 29 Acked 1028 Read 3 Sent Done Done false
31802024-02-07T18:55:46.261Z 30 Acked 1029 Read 9 Sent Done Done false
31812024-02-07T18:55:46.261Z STATES DS:0 DS:1 DS:2 TOTAL
31822024-02-07T18:55:46.261Z New 0 0 0 0
31832024-02-07T18:55:46.261Z Sent 29 0 0 29
31842024-02-07T18:55:46.262Z Done 1 30 30 61
31852024-02-07T18:55:46.262Z Skipped 0 0 0 0
31862024-02-07T18:55:46.262Z Error 0 0 0 0
31872024-02-07T18:55:46.262ZLast Flush: 0 1021 1021
31882024-02-07T18:55:46.262ZDownstairs last five completed:
31892024-02-07T18:55:46.262ZUpstairs last five completed: 30 29 28 27 26
31902024-02-07T18:55:46.262Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:45.108447852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1247}
31912024-02-07T18:55:46.262ZWrote out file "/tmp/test_repair_verify.data"
31922024-02-07T18:55:46.262Z
31932024-02-07T18:55:46.262ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31942024-02-07T18:55:46.262ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31952024-02-07T18:55:46.262Z 0 000-019 25 25 25 48 48 48 F F F
31962024-02-07T18:55:46.262Z 1 020-039 25 25 25 48 48 48 F T T <---
31972024-02-07T18:55:46.262Z 2 040-059 26 26 26 49 49 49 F F F
31982024-02-07T18:55:46.262Z 3 060-079 18 18 18 37 37 37 F F F
31992024-02-07T18:55:46.262Z 4 080-099 26 26 26 50 50 50 F F F
32002024-02-07T18:55:46.262Z 5 100-119 26 28 28 50 53 53 F F F <---
32012024-02-07T18:55:46.262Z 6 120-139 26 26 26 50 50 50 F F F
32022024-02-07T18:55:46.262Z 7 140-159 26 26 26 50 50 50 F F F
32032024-02-07T18:55:46.262Z 8 160-179 27 27 27 51 51 51 F F F
32042024-02-07T18:55:46.262Z 9 180-199 20 28 28 41 53 53 F F F <---
32052024-02-07T18:55:46.263Z 10 200-219 22 22 22 44 44 44 F F F
32062024-02-07T18:55:46.263Z 11 220-239 27 27 27 51 51 51 F F F
32072024-02-07T18:55:46.263Z 12 240-259 26 26 26 49 49 49 F F F
32082024-02-07T18:55:46.263Z 13 260-279 26 26 26 49 49 49 F F F
32092024-02-07T18:55:46.263Z 14 280-299 27 28 28 51 53 53 F F F <---
32102024-02-07T18:55:46.263Z 15 300-319 12 12 12 24 24 24 F T T <---
32112024-02-07T18:55:46.263Z 16 320-339 24 24 24 45 45 45 F F F
32122024-02-07T18:55:46.263Z 17 340-359 24 28 28 45 53 53 T F F <---
32132024-02-07T18:55:46.263Z 18 360-379 22 22 22 44 44 44 F F F
32142024-02-07T18:55:46.263Z 19 380-399 27 27 27 51 51 51 F F F
32152024-02-07T18:55:46.263Z 20 400-419 17 28 28 34 53 53 F F F <---
32162024-02-07T18:55:46.263Z 21 420-439 18 28 28 37 53 53 F F F <---
32172024-02-07T18:55:46.263Z 22 440-459 27 28 28 51 53 53 F F F <---
32182024-02-07T18:55:46.263Z 23 460-479 27 27 27 51 51 51 F F F
32192024-02-07T18:55:46.263Z 24 480-499 27 27 27 51 51 51 F F F
32202024-02-07T18:55:46.264Z 25 500-519 27 27 27 51 51 51 F F F
32212024-02-07T18:55:46.264Z 26 520-539 27 28 28 51 53 53 F F F <---
32222024-02-07T18:55:46.264Z 27 540-559 25 25 25 48 48 48 F F F
32232024-02-07T18:55:46.264Z 28 560-579 27 28 28 51 53 53 F F F <---
32242024-02-07T18:55:46.264Z 29 580-599 18 18 18 37 37 37 F F F
32252024-02-07T18:55:46.264ZMax gen: 28, Max flush: 53
32262024-02-07T18:55:46.268ZError: Difference in extent metadata found!
32272024-02-07T18:55:46.268ZA Difference in extent metadata is expected here
32282024-02-07T18:55:46.268ZOn loop 14
32292024-02-07T18:55:46.268Z
32302024-02-07T18:55:46.268ZVerifying data now
32312024-02-07T18:55:51.427ZNo change after verify
32322024-02-07T18:55:51.427ZLoop: 14 Downstairs dump after verify (and repair):
32332024-02-07T18:55:51.449ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32342024-02-07T18:55:51.449Z 0 000-019 25 25 25 48 48 48 F F F
32352024-02-07T18:55:51.449Z 1 020-039 29 29 29 54 54 54 F F F
32362024-02-07T18:55:51.449Z 2 040-059 26 26 26 49 49 49 F F F
32372024-02-07T18:55:51.449Z 3 060-079 18 18 18 37 37 37 F F F
32382024-02-07T18:55:51.449Z 4 080-099 26 26 26 50 50 50 F F F
32392024-02-07T18:55:51.449Z 5 100-119 28 28 28 53 53 53 F F F
32402024-02-07T18:55:51.452Z 6 120-139 26 26 26 50 50 50 F F F
32412024-02-07T18:55:51.452Z 7 140-159 26 26 26 50 50 50 F F F
32422024-02-07T18:55:51.452Z 8 160-179 27 27 27 51 51 51 F F F
32432024-02-07T18:55:51.452Z 9 180-199 28 28 28 53 53 53 F F F
32442024-02-07T18:55:51.452Z 10 200-219 22 22 22 44 44 44 F F F
32452024-02-07T18:55:51.452Z 11 220-239 27 27 27 51 51 51 F F F
32462024-02-07T18:55:51.452Z 12 240-259 26 26 26 49 49 49 F F F
32472024-02-07T18:55:51.452Z 13 260-279 26 26 26 49 49 49 F F F
32482024-02-07T18:55:51.452Z 14 280-299 28 28 28 53 53 53 F F F
32492024-02-07T18:55:51.452Z 15 300-319 29 29 29 54 54 54 F F F
32502024-02-07T18:55:51.452Z 16 320-339 24 24 24 45 45 45 F F F
32512024-02-07T18:55:51.452Z 17 340-359 28 28 28 53 53 53 F F F
32522024-02-07T18:55:51.452Z 18 360-379 22 22 22 44 44 44 F F F
32532024-02-07T18:55:51.453Z 19 380-399 27 27 27 51 51 51 F F F
32542024-02-07T18:55:51.453Z 20 400-419 28 28 28 53 53 53 F F F
32552024-02-07T18:55:51.453Z 21 420-439 28 28 28 53 53 53 F F F
32562024-02-07T18:55:51.453Z 22 440-459 28 28 28 53 53 53 F F F
32572024-02-07T18:55:51.453Z 23 460-479 27 27 27 51 51 51 F F F
32582024-02-07T18:55:51.453Z 24 480-499 27 27 27 51 51 51 F F F
32592024-02-07T18:55:51.453Z 25 500-519 27 27 27 51 51 51 F F F
32602024-02-07T18:55:51.453Z 26 520-539 28 28 28 53 53 53 F F F
32612024-02-07T18:55:51.453Z 27 540-559 25 25 25 48 48 48 F F F
32622024-02-07T18:55:51.453Z 28 560-579 28 28 28 53 53 53 F F F
32632024-02-07T18:55:51.453Z 29 580-599 18 18 18 37 37 37 F F F
32642024-02-07T18:55:51.453ZMax gen: 29, Max flush: 54
32652024-02-07T18:55:51.453Z
32662024-02-07T18:55:51.453ZBegin loop 15
32672024-02-07T18:55:51.453ZDownstairs to restart: 2
32682024-02-07T18:55:51.464Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728334589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255}
32692024-02-07T18:55:51.464Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728582385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255}
32702024-02-07T18:55:51.468Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.72862046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255}
32712024-02-07T18:55:51.468Z{"msg":"Crucible 8ee74346-c7a2-40b4-b777-ffed6d3e6cc2 has session id: cb9f2c27-e3c7-4405-af24-ea32254b0130","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728645547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32722024-02-07T18:55:51.469Z{"msg":"Upstairs opts: Upstairs UUID: 8ee74346-c7a2-40b4-b777-ffed6d3e6cc2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
32732024-02-07T18:55:51.469Z,"time":"2024-02-07T18:55:50.728689021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32742024-02-07T18:55:51.469Z{"msg":"Crucible stats registered with UUID: 8ee74346-c7a2-40b4-b777-ffed6d3e6cc2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728719637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32752024-02-07T18:55:51.469Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728752752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255}
32762024-02-07T18:55:51.469Z{"msg":"8ee74346-c7a2-40b4-b777-ffed6d3e6cc2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728827342Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32772024-02-07T18:55:51.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728855208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32782024-02-07T18:55:51.469Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728882065Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32792024-02-07T18:55:51.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.72891807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32802024-02-07T18:55:51.469Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.728946156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32812024-02-07T18:55:51.469Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.72899141Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32822024-02-07T18:55:51.469Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729018006Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32832024-02-07T18:55:51.469Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729053651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32842024-02-07T18:55:51.469Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729092546Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32852024-02-07T18:55:51.469Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729167126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32862024-02-07T18:55:51.469Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:50.729206131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32872024-02-07T18:55:51.469Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:50.729251095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32882024-02-07T18:55:51.469Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729279861Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32892024-02-07T18:55:51.469Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:50.729306077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32902024-02-07T18:55:51.469Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729329604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32912024-02-07T18:55:51.469Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729367709Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32922024-02-07T18:55:51.469Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729396025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32932024-02-07T18:55:51.469Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.72943212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32942024-02-07T18:55:51.469Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729918665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32952024-02-07T18:55:51.469Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729947191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32962024-02-07T18:55:51.470Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.729962888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32972024-02-07T18:55:51.470Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.730001143Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32982024-02-07T18:55:51.470Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.730558728Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
32992024-02-07T18:55:51.470Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.730595383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33002024-02-07T18:55:51.470Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.731200841Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33012024-02-07T18:55:51.470Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.731236766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33022024-02-07T18:55:51.470Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.731252234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33032024-02-07T18:55:51.470Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.731272442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33042024-02-07T18:55:51.470Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.731286769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33052024-02-07T18:55:51.470Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:50.731301138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33062024-02-07T18:55:52.468Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.730306962Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33072024-02-07T18:55:52.468Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.73046714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33082024-02-07T18:55:52.480Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.731309746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33092024-02-07T18:55:52.480Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.731346001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33102024-02-07T18:55:52.480Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.731836625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33112024-02-07T18:55:52.480Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732457961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33122024-02-07T18:55:52.480Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732497306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33132024-02-07T18:55:52.480Z{"msg":"[0]R flush_numbers[0..12]: [48, 54, 49, 37, 50, 53, 50, 50, 51, 53, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732515293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33142024-02-07T18:55:52.480Z{"msg":"[0]R generation[0..12]: [25, 29, 26, 18, 26, 28, 26, 26, 27, 28, 22, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.73253804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33152024-02-07T18:55:52.480Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732553208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33162024-02-07T18:55:52.480Z{"msg":"[1]R flush_numbers[0..12]: [48, 54, 49, 37, 50, 53, 50, 50, 51, 53, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732579764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33172024-02-07T18:55:52.480Z{"msg":"[1]R generation[0..12]: [25, 29, 26, 18, 26, 28, 26, 26, 27, 28, 22, 27]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
33182024-02-07T18:55:52.480Z,"time":"2024-02-07T18:55:51.732602012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33192024-02-07T18:55:52.480Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732621479Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33202024-02-07T18:55:52.480Z{"msg":"[2]R flush_numbers[0..12]: [48, 54, 49, 37, 50, 53, 50, 50, 51, 53, 44, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732643886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33212024-02-07T18:55:52.480Z{"msg":"[2]R generation[0..12]: [25, 29, 26, 18, 26, 28, 26, 26, 27, 28, 22, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732658704Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33222024-02-07T18:55:52.481Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.73268731Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33232024-02-07T18:55:52.481Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732719776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33242024-02-07T18:55:52.481Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732735793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33252024-02-07T18:55:52.481Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732768689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33262024-02-07T18:55:52.481Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732803224Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33272024-02-07T18:55:52.481Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732823602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33282024-02-07T18:55:52.481Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732856627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33292024-02-07T18:55:52.481Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732879354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33302024-02-07T18:55:52.481Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732912659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33312024-02-07T18:55:52.481Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732931817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33322024-02-07T18:55:52.481Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732963463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33332024-02-07T18:55:52.481Z{"msg":"8ee74346-c7a2-40b4-b777-ffed6d3e6cc2 is now active with session: cb9f2c27-e3c7-4405-af24-ea32254b0130","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.732999708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33342024-02-07T18:55:52.481Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33352024-02-07T18:55:52.481ZSet Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.733037493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255,"session_id":"cb9f2c27-e3c7-4405-af24-ea32254b0130"}
33362024-02-07T18:55:52.481Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:51.73305579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255}
33372024-02-07T18:55:52.481ZLoading write count information from file "/tmp/test_repair_verify.data"
33382024-02-07T18:55:52.481ZRun Repair workload
33392024-02-07T18:55:52.481Z01/30 Write block 508 len 32768 data: 3 3 3 3 5 5 5 4
33402024-02-07T18:55:52.481Z02/30 Read block 535 len 40960
33412024-02-07T18:55:52.485Z03/30 Read block 284 len 4096
33422024-02-07T18:55:52.485Z04/30 Write block 494 len 12288 data: 5 4 4
33432024-02-07T18:55:52.485Z05/30 Flush
33442024-02-07T18:55:52.485Z06/30 Write block 428 len 40960 data: 6 5 4 4 2 2 2 3 4 5
33452024-02-07T18:55:52.485Z07/30 Read block 187 len 28672
33462024-02-07T18:55:52.485Z08/30 Write block 315 len 16384 data: 4 5 4 4
33472024-02-07T18:55:52.486Z09/30 Read block 455 len 8192
33482024-02-07T18:55:52.486Z10/30 Read block 398 len 12288
33492024-02-07T18:55:52.486Z11/30 Read block 79 len 8192
33502024-02-07T18:55:52.486Z12/30 Read block 315 len 36864
33512024-02-07T18:55:52.486Z13/30 Write block 55 len 32768 data: 6 6 6 5 4 3 3 3
33522024-02-07T18:55:52.486Z14/30 Read block 470 len 28672
33532024-02-07T18:55:52.486Z15/30 Write block 151 len 12288 data: 3 3 3
33542024-02-07T18:55:52.486Z16/30 Read block 246 len 12288
33552024-02-07T18:55:52.492Z17/30 Read block 254 len 20480
33562024-02-07T18:55:52.492Z18/30 Read block 163 len 20480
33572024-02-07T18:55:52.492Z19/30 Read block 237 len 4096
33582024-02-07T18:55:52.492Z20/30 Read block 9 len 20480
33592024-02-07T18:55:52.495Z21/30 Flush
33602024-02-07T18:55:52.495Z22/30 Write block 224 len 40960 data: 5 3 2 2 2 2 3 3 4 5
33612024-02-07T18:55:52.495Z23/30 Write block 198 len 4096 data: 5
33622024-02-07T18:55:52.495Z24/30 Write block 16 len 40960 data: 4 5 4 5 5 5 5 6 6 5
33632024-02-07T18:55:52.498Z25/30 Write block 491 len 16384 data: 4 4 5 6
33642024-02-07T18:55:52.498Z26/30 Read block 226 len 20480
33652024-02-07T18:55:52.498Z27/30 Read block 402 len 12288
33662024-02-07T18:55:52.498Z28/30 Read block 160 len 4096
33672024-02-07T18:55:52.501Z29/30 Write block 180 len 36864 data: 5 3 3 3 2 3 3 3 3
33682024-02-07T18:55:52.501Z30/30 Write block 499 len 32768 data: 3 3 2 2 3 3 3 2
33692024-02-07T18:55:52.501Z----------------------------------------------------------------
33702024-02-07T18:55:52.502Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33712024-02-07T18:55:52.502ZStates: Active Active Active
33722024-02-07T18:55:52.502ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33732024-02-07T18:55:52.502Z 1 Acked 1000 Write 8 Done Done Sent false
33742024-02-07T18:55:52.502Z 2 Acked 1001 Read 10 Done Done Sent false
33752024-02-07T18:55:52.502Z 3 Acked 1002 Read 1 Done Done Sent false
33762024-02-07T18:55:52.502Z 4 Acked 1003 Write 3 Done Done Sent false
33772024-02-07T18:55:52.502Z 5 Acked 1004 Flush 0 Done Done Sent false
33782024-02-07T18:55:52.502Z 6 Acked 1005 Write 10 Done Done Sent false
33792024-02-07T18:55:52.502Z 7 Acked 1006 Read 7 Done Done Sent false
33802024-02-07T18:55:52.502Z 8 Acked 1007 Write 4 Done Done Sent false
33812024-02-07T18:55:52.502Z 9 Acked 1008 Read 2 Done Done Sent false
33822024-02-07T18:55:52.502Z 10 Acked 1009 Read 3 Done Done Sent false
33832024-02-07T18:55:52.502Z 11 Acked 1010 Read 2 Done Done Sent false
33842024-02-07T18:55:52.502Z 12 Acked 1011 Read 9 Done Done Sent false
33852024-02-07T18:55:52.502Z 13 Acked 1012 Write 8 Done Done Sent false
33862024-02-07T18:55:52.502Z 14 Acked 1013 Read 7 Done Done Sent false
33872024-02-07T18:55:52.502Z 15 Acked 1014 Write 3 Done Done Sent false
33882024-02-07T18:55:52.502Z 16 Acked 1015 Read 3 Done Done Sent false
33892024-02-07T18:55:52.502Z 17 Acked 1016 Read 5 Done Done Sent false
33902024-02-07T18:55:52.502Z 18 Acked 1017 Read 5 Done Done Sent false
33912024-02-07T18:55:52.502Z 19 Acked 1018 Read 1 Done Done Sent false
33922024-02-07T18:55:52.502Z 20 Acked 1019 Read 5 Done Done Sent false
33932024-02-07T18:55:52.502Z 21 Acked 1020 Flush 0 Done Done Sent false
33942024-02-07T18:55:52.502Z 22 Acked 1021 Write 10 Done Done Sent false
33952024-02-07T18:55:52.503Z 23 Acked 1022 Write 1 Done Done Sent false
33962024-02-07T18:55:52.503Z 24 Acked 1023 Write 10 Done Done Sent false
33972024-02-07T18:55:52.503Z 25 Acked 1024 Write 4 Done Done Sent false
33982024-02-07T18:55:52.503Z 26 Acked 1025 Read 5 Done Done Sent false
33992024-02-07T18:55:52.503Z 27 Acked 1026 Read 3 Done Done Sent false
34002024-02-07T18:55:52.503Z 28 Acked 1027 Read 1 Done Done Sent false
34012024-02-07T18:55:52.503Z 29 Acked 1028 Write 9 Sent Done Sent false
34022024-02-07T18:55:52.503Z 30 Acked 1029 Write 8 Sent Sent Sent false
34032024-02-07T18:55:52.503Z STATES DS:0 DS:1 DS:2 TOTAL
34042024-02-07T18:55:52.503Z New 0 0 0 0
34052024-02-07T18:55:52.503Z Sent 2 1 30 33
34062024-02-07T18:55:52.503Z Done 28 29 0 57
34072024-02-07T18:55:52.503Z Skipped 0 0 0 0
34082024-02-07T18:55:52.503Z Error 0 0 0 0
34092024-02-07T18:55:52.503ZLast Flush: 1020 1020 0
34102024-02-07T18:55:52.503ZDownstairs last five completed:
34112024-02-07T18:55:52.503ZUpstairs last five completed: 30 29 28 27 26
34122024-02-07T18:55:52.503Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:51.764509546Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1255}
34132024-02-07T18:55:52.503ZWrote out file "/tmp/test_repair_verify.data"
34142024-02-07T18:55:52.506Z
34152024-02-07T18:55:52.509ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
34162024-02-07T18:55:52.526ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34172024-02-07T18:55:52.526Z 0 000-019 25 25 25 48 48 48 T T F <---
34182024-02-07T18:55:52.526Z 1 020-039 29 29 29 54 54 54 T T F <---
34192024-02-07T18:55:52.526Z 2 040-059 30 30 26 56 56 49 F F F <---
34202024-02-07T18:55:52.526Z 3 060-079 30 30 18 56 56 37 F F F <---
34212024-02-07T18:55:52.526Z 4 080-099 26 26 26 50 50 50 F F F
34222024-02-07T18:55:52.526Z 5 100-119 28 28 28 53 53 53 F F F
34232024-02-07T18:55:52.526Z 6 120-139 26 26 26 50 50 50 F F F
34242024-02-07T18:55:52.526Z 7 140-159 30 30 26 56 56 50 F F F <---
34252024-02-07T18:55:52.526Z 8 160-179 27 27 27 51 51 51 F F F
34262024-02-07T18:55:52.527Z 9 180-199 28 28 28 53 53 53 T T F <---
34272024-02-07T18:55:52.527Z 10 200-219 22 22 22 44 44 44 F F F
34282024-02-07T18:55:52.527Z 11 220-239 27 27 27 51 51 51 T T F <---
34292024-02-07T18:55:52.527Z 12 240-259 26 26 26 49 49 49 F F F
34302024-02-07T18:55:52.527Z 13 260-279 26 26 26 49 49 49 F F F
34312024-02-07T18:55:52.530Z 14 280-299 28 28 28 53 53 53 F F F
34322024-02-07T18:55:52.530Z 15 300-319 30 30 29 56 56 54 F F F <---
34332024-02-07T18:55:52.530Z 16 320-339 24 24 24 45 45 45 F F F
34342024-02-07T18:55:52.530Z 17 340-359 28 28 28 53 53 53 F F F
34352024-02-07T18:55:52.530Z 18 360-379 22 22 22 44 44 44 F F F
34362024-02-07T18:55:52.530Z 19 380-399 27 27 27 51 51 51 F F F
34372024-02-07T18:55:52.530Z 20 400-419 28 28 28 53 53 53 F F F
34382024-02-07T18:55:52.530Z 21 420-439 30 30 28 56 56 53 F F F <---
34392024-02-07T18:55:52.530Z 22 440-459 28 28 28 53 53 53 F F F
34402024-02-07T18:55:52.530Z 23 460-479 27 27 27 51 51 51 F F F
34412024-02-07T18:55:52.530Z 24 480-499 30 30 27 55 55 51 T T F <---
34422024-02-07T18:55:52.531Z 25 500-519 30 30 27 55 55 51 T T F <---
34432024-02-07T18:55:52.531Z 26 520-539 28 28 28 53 53 53 F F F
34442024-02-07T18:55:52.531Z 27 540-559 25 25 25 48 48 48 F F F
34452024-02-07T18:55:52.531Z 28 560-579 28 28 28 53 53 53 F F F
34462024-02-07T18:55:52.531Z 29 580-599 18 18 18 37 37 37 F F F
34472024-02-07T18:55:52.531ZMax gen: 30, Max flush: 56
34482024-02-07T18:55:52.531ZError: Difference in extent metadata found!
34492024-02-07T18:55:52.531ZA Difference in extent metadata is expected here
34502024-02-07T18:55:52.531ZOn loop 15
34512024-02-07T18:55:52.531Z
34522024-02-07T18:55:52.534ZVerifying data now
34532024-02-07T18:55:57.113ZNo change after verify
34542024-02-07T18:55:57.113ZLoop: 15 Downstairs dump after verify (and repair):
34552024-02-07T18:55:57.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34562024-02-07T18:55:57.128Z 0 000-019 31 31 31 57 57 57 F F F
34572024-02-07T18:55:57.128Z 1 020-039 31 31 31 57 57 57 F F F
34582024-02-07T18:55:57.128Z 2 040-059 30 30 30 56 56 56 F F F
34592024-02-07T18:55:57.128Z 3 060-079 30 30 30 56 56 56 F F F
34602024-02-07T18:55:57.129Z 4 080-099 26 26 26 50 50 50 F F F
34612024-02-07T18:55:57.129Z 5 100-119 28 28 28 53 53 53 F F F
34622024-02-07T18:55:57.129Z 6 120-139 26 26 26 50 50 50 F F F
34632024-02-07T18:55:57.129Z 7 140-159 30 30 30 56 56 56 F F F
34642024-02-07T18:55:57.129Z 8 160-179 27 27 27 51 51 51 F F F
34652024-02-07T18:55:57.129Z 9 180-199 31 31 31 57 57 57 F F F
34662024-02-07T18:55:57.129Z 10 200-219 22 22 22 44 44 44 F F F
34672024-02-07T18:55:57.129Z 11 220-239 31 31 31 57 57 57 F F F
34682024-02-07T18:55:57.129Z 12 240-259 26 26 26 49 49 49 F F F
34692024-02-07T18:55:57.133Z 13 260-279 26 26 26 49 49 49 F F F
34702024-02-07T18:55:57.133Z 14 280-299 28 28 28 53 53 53 F F F
34712024-02-07T18:55:57.133Z 15 300-319 30 30 30 56 56 56 F F F
34722024-02-07T18:55:57.133Z 16 320-339 24 24 24 45 45 45 F F F
34732024-02-07T18:55:57.133Z 17 340-359 28 28 28 53 53 53 F F F
34742024-02-07T18:55:57.133Z 18 360-379 22 22 22 44 44 44 F F F
34752024-02-07T18:55:57.133Z 19 380-399 27 27 27 51 51 51 F F F
34762024-02-07T18:55:57.133Z 20 400-419 28 28 28 53 53 53 F F F
34772024-02-07T18:55:57.134Z 21 420-439 30 30 30 56 56 56 F F F
34782024-02-07T18:55:57.134Z 22 440-459 28 28 28 53 53 53 F F F
34792024-02-07T18:55:57.134Z 23 460-479 27 27 27 51 51 51 F F F
34802024-02-07T18:55:57.134Z 24 480-499 31 31 31 57 57 57 F F F
34812024-02-07T18:55:57.134Z 25 500-519 31 31 31 57 57 57 F F F
34822024-02-07T18:55:57.134Z 26 520-539 28 28 28 53 53 53 F F F
34832024-02-07T18:55:57.134Z 27 540-559 25 25 25 48 48 48 F F F
34842024-02-07T18:55:57.134Z 28 560-579 28 28 28 53 53 53 F F F
34852024-02-07T18:55:57.134Z 29 580-599 18 18 18 37 37 37 F F F
34862024-02-07T18:55:57.134ZMax gen: 31, Max flush: 57
34872024-02-07T18:55:57.134Z
34882024-02-07T18:55:57.134ZBegin loop 16
34892024-02-07T18:55:57.134ZDownstairs to restart: 0
34902024-02-07T18:55:57.152Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413058248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263}
34912024-02-07T18:55:57.152Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413319783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263}
34922024-02-07T18:55:57.152Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.41333908Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263}
34932024-02-07T18:55:57.152Z{"msg":"Crucible 5d078172-7e76-4024-bb13-f6a2313843c8 has session id: 274e8b51-40e8-496e-9455-47aa37dd933f","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413358427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
34942024-02-07T18:55:57.152Z{"msg":"Upstairs opts: Upstairs UUID: 5d078172-7e76-4024-bb13-f6a2313843c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413394302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
34952024-02-07T18:55:57.165Z{"msg":"Crucible stats registered with UUID: 5d078172-7e76-4024-bb13-f6a2313843c8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413419459Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
34962024-02-07T18:55:57.166ZCrucible runtime is spawned
34972024-02-07T18:55:57.166Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413591656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263}
34982024-02-07T18:55:57.166Z{"msg":"5d078172-7e76-4024-bb13-f6a2313843c8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413622212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
34992024-02-07T18:55:57.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.4136368Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35002024-02-07T18:55:57.166Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413651788Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35012024-02-07T18:55:57.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413679404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35022024-02-07T18:55:57.167Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413699681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35032024-02-07T18:55:57.167Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413722078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35042024-02-07T18:55:57.167Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413746355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35052024-02-07T18:55:57.167Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413770402Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35062024-02-07T18:55:57.167Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413810636Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35072024-02-07T18:55:57.167Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413834083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35082024-02-07T18:55:57.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.413977234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35092024-02-07T18:55:57.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414014609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35102024-02-07T18:55:57.167Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:56.414045574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35112024-02-07T18:55:57.167Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:56.414088189Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35122024-02-07T18:55:57.167Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:56.414117015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35132024-02-07T18:55:57.167Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.4141542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35142024-02-07T18:55:57.168Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414176117Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35152024-02-07T18:55:57.168Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414316778Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35162024-02-07T18:55:57.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414734161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35172024-02-07T18:55:57.168Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414769836Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35182024-02-07T18:55:57.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414786474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35192024-02-07T18:55:57.168Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.414810521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35202024-02-07T18:55:57.168Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.415394072Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35212024-02-07T18:55:57.168Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.415510076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35222024-02-07T18:55:57.168Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.41599829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35232024-02-07T18:55:57.168Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.416039215Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35242024-02-07T18:55:57.168Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.416055543Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35252024-02-07T18:55:57.168Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.4160759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35262024-02-07T18:55:57.168Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.416090048Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35272024-02-07T18:55:57.169Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:56.416111055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35282024-02-07T18:55:58.160Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.416123903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35292024-02-07T18:55:58.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.416305498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35302024-02-07T18:55:58.165Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.417211436Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35312024-02-07T18:55:58.165Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.417259119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35322024-02-07T18:55:58.165Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.4177695Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35332024-02-07T18:55:58.165Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.41835996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35342024-02-07T18:55:58.165Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418395735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35352024-02-07T18:55:58.165Z{"msg":"[0]R flush_numbers[0..12]: [57, 57, 56, 56, 50, 53, 50, 56, 51, 57, 44, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418415583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35362024-02-07T18:55:58.165Z{"msg":"[0]R generation[0..12]: [31, 31, 30, 30, 26, 28, 26, 30, 27, 31, 22, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.41843368Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35372024-02-07T18:55:58.165Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418461027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35382024-02-07T18:55:58.165Z{"msg":"[1]R flush_numbers[0..12]: [57, 57, 56, 56, 50, 53, 50, 56, 51, 57, 44, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418479374Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35392024-02-07T18:55:58.165Z{"msg":"[1]R generation[0..12]: [31, 31, 30, 30, 26, 28, 26, 30, 27, 31, 22, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418504351Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35402024-02-07T18:55:58.165Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418522448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35412024-02-07T18:55:58.165Z{"msg":"[2]R flush_numbers[0..12]: [57, 57, 56, 56, 50, 53, 50, 56, 51, 57, 44, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418548235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35422024-02-07T18:55:58.165Z{"msg":"[2]R generation[0..12]: [31, 31, 30, 30, 26, 28, 26, 30, 27, 31, 22, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418565663Z",Wait for a query_work_queue command to finish before sending IO
35432024-02-07T18:55:58.166Z"hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35442024-02-07T18:55:58.166Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418596648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35452024-02-07T18:55:58.166Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418614226Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35462024-02-07T18:55:58.166Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418639633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35472024-02-07T18:55:58.166Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.41865642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35482024-02-07T18:55:58.166Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418681367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35492024-02-07T18:55:58.166Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418699205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35502024-02-07T18:55:58.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418722481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35512024-02-07T18:55:58.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418739379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35522024-02-07T18:55:58.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418763976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35532024-02-07T18:55:58.166Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418782543Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35542024-02-07T18:55:58.166Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.41880707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35552024-02-07T18:55:58.166Z{"msg":"5d078172-7e76-4024-bb13-f6a2313843c8 is now active with session: 274e8b51-40e8-496e-9455-47aa37dd933f","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418821498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35562024-02-07T18:55:58.166Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418833826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263,"session_id":"274e8b51-40e8-496e-9455-47aa37dd933f"}
35572024-02-07T18:55:58.166Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:55:57.418852564Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263}
35582024-02-07T18:55:58.166ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35592024-02-07T18:55:58.166ZLoading write count information from file "/tmp/test_repair_verify.data"
35602024-02-07T18:55:58.166ZRun Repair workload
35612024-02-07T18:55:58.166Z01/30 Write block 208 len 4096 data: 3
35622024-02-07T18:55:58.166Z02/30 Write block 524 len 4096 data: 2
35632024-02-07T18:55:58.166Z03/30 Read block 185 len 16384
35642024-02-07T18:55:58.166Z04/30 Write block 388 len 40960 data: 4 5 4 3 4 5 5 5 5 5
35652024-02-07T18:55:58.166Z05/30 Read block 157 len 24576
35662024-02-07T18:55:58.166Z06/30 Read block 445 len 16384
35672024-02-07T18:55:58.167Z07/30 Flush
35682024-02-07T18:55:58.170Z08/30 Write block 502 len 36864 data: 3 4 4 4 3 3 4 4 4
35692024-02-07T18:55:58.170Z09/30 Write block 293 len 20480 data: 3 2 2 2 2
35702024-02-07T18:55:58.170Z10/30 Write block 97 len 36864 data: 4 3 3 2 2 2 2 4 4
35712024-02-07T18:55:58.170Z11/30 Flush
35722024-02-07T18:55:58.170Z12/30 Write block 575 len 20480 data: 3 3 3 2 2
35732024-02-07T18:55:58.173Z13/30 Write block 345 len 28672 data: 5 6 4 4 4 5 4
35742024-02-07T18:55:58.173Z14/30 Write block 201 len 20480 data: 2 3 4 4 4
35752024-02-07T18:55:58.173Z15/30 Read block 503 len 24576
35762024-02-07T18:55:58.176Z16/30 Write block 191 len 16384 data: 5 5 4 4
35772024-02-07T18:55:58.176Z17/30 Read block 327 len 32768
35782024-02-07T18:55:58.176Z18/30 Write block 581 len 24576 data: 3 3 3 3 3 4
35792024-02-07T18:55:58.176Z19/30 Write block 537 len 24576 data: 2 2 2 2 3 4
35802024-02-07T18:55:58.176Z20/30 Flush
35812024-02-07T18:55:58.180Z21/30 Write block 259 len 28672 data: 4 4 4 4 4 4 3
35822024-02-07T18:55:58.183Z22/30 Write block 368 len 4096 data: 4
35832024-02-07T18:55:58.183Z23/30 Write block 186 len 8192 data: 4 4
35842024-02-07T18:55:58.183Z24/30 Write block 291 len 32768 data: 4 3 4 3 3 3 3 2
35852024-02-07T18:55:58.183Z25/30 Read block 84 len 36864
35862024-02-07T18:55:58.186Z26/30 Write block 304 len 32768 data: 3 3 3 3 3 3 3 3
35872024-02-07T18:55:58.186Z27/30 Read block 596 len 12288
35882024-02-07T18:55:58.186Z28/30 Read block 302 len 24576
35892024-02-07T18:55:58.190Z29/30 Read block 434 len 20480
35902024-02-07T18:55:58.190Z30/30 Read block 368 len 4096
35912024-02-07T18:55:58.190Z----------------------------------------------------------------
35922024-02-07T18:55:58.190Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35932024-02-07T18:55:58.190ZStates: Active Active Active
35942024-02-07T18:55:58.190ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35952024-02-07T18:55:58.190Z 1 Acked 1000 Write 1 Sent Done Done false
35962024-02-07T18:55:58.190Z 2 Acked 1001 Write 1 Sent Done Done false
35972024-02-07T18:55:58.190Z 3 Acked 1002 Read 4 Sent Done Done false
35982024-02-07T18:55:58.190Z 4 Acked 1003 Write 10 Sent Done Done false
35992024-02-07T18:55:58.190Z 5 Acked 1004 Read 6 Sent Done Done false
36002024-02-07T18:55:58.190Z 6 Acked 1005 Read 4 Sent Done Done false
36012024-02-07T18:55:58.190Z 7 Acked 1006 Flush 0 Sent Done Done false
36022024-02-07T18:55:58.190Z 8 Acked 1007 Write 9 Sent Done Done false
36032024-02-07T18:55:58.190Z 9 Acked 1008 Write 5 Sent Done Done false
36042024-02-07T18:55:58.190Z 10 Acked 1009 Write 9 Sent Done Done false
36052024-02-07T18:55:58.190Z 11 Acked 1010 Flush 0 Sent Done Done false
36062024-02-07T18:55:58.190Z 12 Acked 1011 Write 5 Sent Done Done false
36072024-02-07T18:55:58.190Z 13 Acked 1012 Write 7 Sent Done Done false
36082024-02-07T18:55:58.190Z 14 Acked 1013 Write 5 Sent Done Done false
36092024-02-07T18:55:58.191Z 15 Acked 1014 Read 6 Sent Done Done false
36102024-02-07T18:55:58.191Z 16 Acked 1015 Write 4 Sent Done Done false
36112024-02-07T18:55:58.191Z 17 Acked 1016 Read 8 Sent Done Done false
36122024-02-07T18:55:58.191Z 18 Acked 1017 Write 6 Sent Done Done false
36132024-02-07T18:55:58.191Z 19 Acked 1018 Write 6 Sent Done Done false
36142024-02-07T18:55:58.191Z 20 Acked 1019 Flush 0 Sent Done Done false
36152024-02-07T18:55:58.191Z 21 Acked 1020 Write 7 Sent Done Done false
36162024-02-07T18:55:58.191Z 22 Acked 1021 Write 1 Sent Done Done false
36172024-02-07T18:55:58.191Z 23 Acked 1022 Write 2 Sent Done Done false
36182024-02-07T18:55:58.191Z 24 Acked 1023 Write 8 Sent Done Done false
36192024-02-07T18:55:58.191Z 25 Acked 1024 Read 9 Sent Done Done false
36202024-02-07T18:55:58.191Z 26 Acked 1025 Write 8 Sent Done Done false
36212024-02-07T18:55:58.191Z 27 Acked 1026 Read 3 Sent Done Done false
36222024-02-07T18:55:58.191Z 28 Acked 1027 Read 6 Sent Done Done false
36232024-02-07T18:55:58.191Z 29 Acked 1028 Read 5 Sent Done Done false
36242024-02-07T18:55:58.191Z 30 Acked 1029 Read 1 Sent Done Done false
36252024-02-07T18:55:58.191Z STATES DS:0 DS:1 DS:2 TOTAL
36262024-02-07T18:55:58.191Z New 0 0 0 0
36272024-02-07T18:55:58.191Z Sent 30 0 0 30
36282024-02-07T18:55:58.191Z Done 0 30 30 60
36292024-02-07T18:55:58.191Z Skipped 0 0 0 0
36302024-02-07T18:55:58.191Z Error 0 0 0 0
36312024-02-07T18:55:58.191ZLast Flush: 0 1019 1019
36322024-02-07T18:55:58.192ZDownstairs last five completed:
36332024-02-07T18:55:58.192ZUpstairs last five completed: 30 29 28 27 26
36342024-02-07T18:55:58.192Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:55:57.451663526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1263}
36352024-02-07T18:55:58.192ZWrote out file "/tmp/test_repair_verify.data"
36362024-02-07T18:55:58.195Z
36372024-02-07T18:55:58.198ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36382024-02-07T18:55:58.213ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36392024-02-07T18:55:58.214Z 0 000-019 31 31 31 57 57 57 F F F
36402024-02-07T18:55:58.214Z 1 020-039 31 31 31 57 57 57 F F F
36412024-02-07T18:55:58.214Z 2 040-059 30 30 30 56 56 56 F F F
36422024-02-07T18:55:58.214Z 3 060-079 30 30 30 56 56 56 F F F
36432024-02-07T18:55:58.214Z 4 080-099 26 32 32 50 59 59 F F F <---
36442024-02-07T18:55:58.214Z 5 100-119 28 32 32 53 59 59 F F F <---
36452024-02-07T18:55:58.214Z 6 120-139 26 26 26 50 50 50 F F F
36462024-02-07T18:55:58.214Z 7 140-159 30 30 30 56 56 56 F F F
36472024-02-07T18:55:58.214Z 8 160-179 27 27 27 51 51 51 F F F
36482024-02-07T18:55:58.217Z 9 180-199 31 32 32 57 60 60 F T T <---
36492024-02-07T18:55:58.217Z 10 200-219 22 32 32 44 60 60 F F F <---
36502024-02-07T18:55:58.217Z 11 220-239 31 31 31 57 57 57 F F F
36512024-02-07T18:55:58.217Z 12 240-259 26 26 26 49 49 49 F T T <---
36522024-02-07T18:55:58.217Z 13 260-279 26 26 26 49 49 49 F T T <---
36532024-02-07T18:55:58.217Z 14 280-299 28 32 32 53 59 59 F T T <---
36542024-02-07T18:55:58.217Z 15 300-319 30 30 30 56 56 56 F T T <---
36552024-02-07T18:55:58.217Z 16 320-339 24 24 24 45 45 45 F F F
36562024-02-07T18:55:58.217Z 17 340-359 28 32 32 53 60 60 F F F <---
36572024-02-07T18:55:58.217Z 18 360-379 22 22 22 44 44 44 F T T <---
36582024-02-07T18:55:58.217Z 19 380-399 27 32 32 51 58 58 F F F <---
36592024-02-07T18:55:58.218Z 20 400-419 28 28 28 53 53 53 F F F
36602024-02-07T18:55:58.218Z 21 420-439 30 30 30 56 56 56 F F F
36612024-02-07T18:55:58.218Z 22 440-459 28 28 28 53 53 53 F F F
36622024-02-07T18:55:58.218Z 23 460-479 27 27 27 51 51 51 F F F
36632024-02-07T18:55:58.218Z 24 480-499 31 31 31 57 57 57 F F F
36642024-02-07T18:55:58.218Z 25 500-519 31 32 32 57 59 59 F F F <---
36652024-02-07T18:55:58.218Z 26 520-539 28 32 32 53 60 60 F F F <---
36662024-02-07T18:55:58.218Z 27 540-559 25 32 32 48 60 60 F F F <---
36672024-02-07T18:55:58.218Z 28 560-579 28 32 32 53 60 60 F F F <---
36682024-02-07T18:55:58.218Z 29 580-599 18 32 32 37 60 60 F F F <---
36692024-02-07T18:55:58.218ZMax gen: 32, Max flush: 60
36702024-02-07T18:55:58.218ZError: Difference in extent metadata found!
36712024-02-07T18:55:58.218ZA Difference in extent metadata is expected here
36722024-02-07T18:55:58.218ZOn loop 16
36732024-02-07T18:55:58.218Z
36742024-02-07T18:55:58.221ZVerifying data now
36752024-02-07T18:56:02.969ZNo change after verify
36762024-02-07T18:56:02.969ZLoop: 16 Downstairs dump after verify (and repair):
36772024-02-07T18:56:02.985ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36782024-02-07T18:56:02.986Z 0 000-019 31 31 31 57 57 57 F F F
36792024-02-07T18:56:02.986Z 1 020-039 31 31 31 57 57 57 F F F
36802024-02-07T18:56:02.986Z 2 040-059 30 30 30 56 56 56 F F F
36812024-02-07T18:56:02.986Z 3 060-079 30 30 30 56 56 56 F F F
36822024-02-07T18:56:02.986Z 4 080-099 32 32 32 59 59 59 F F F
36832024-02-07T18:56:02.986Z 5 100-119 32 32 32 59 59 59 F F F
36842024-02-07T18:56:02.986Z 6 120-139 26 26 26 50 50 50 F F F
36852024-02-07T18:56:02.986Z 7 140-159 30 30 30 56 56 56 F F F
36862024-02-07T18:56:02.986Z 8 160-179 27 27 27 51 51 51 F F F
36872024-02-07T18:56:02.994Z 9 180-199 33 33 33 61 61 61 F F F
36882024-02-07T18:56:02.994Z 10 200-219 32 32 32 60 60 60 F F F
36892024-02-07T18:56:02.994Z 11 220-239 31 31 31 57 57 57 F F F
36902024-02-07T18:56:02.994Z 12 240-259 33 33 33 61 61 61 F F F
36912024-02-07T18:56:02.994Z 13 260-279 33 33 33 61 61 61 F F F
36922024-02-07T18:56:02.994Z 14 280-299 33 33 33 61 61 61 F F F
36932024-02-07T18:56:02.994Z 15 300-319 33 33 33 61 61 61 F F F
36942024-02-07T18:56:02.994Z 16 320-339 24 24 24 45 45 45 F F F
36952024-02-07T18:56:02.994Z 17 340-359 32 32 32 60 60 60 F F F
36962024-02-07T18:56:02.994Z 18 360-379 33 33 33 61 61 61 F F F
36972024-02-07T18:56:02.994Z 19 380-399 32 32 32 58 58 58 F F F
36982024-02-07T18:56:02.994Z 20 400-419 28 28 28 53 53 53 F F F
36992024-02-07T18:56:02.994Z 21 420-439 30 30 30 56 56 56 F F F
37002024-02-07T18:56:02.994Z 22 440-459 28 28 28 53 53 53 F F F
37012024-02-07T18:56:02.994Z 23 460-479 27 27 27 51 51 51 F F F
37022024-02-07T18:56:02.994Z 24 480-499 31 31 31 57 57 57 F F F
37032024-02-07T18:56:02.995Z 25 500-519 32 32 32 59 59 59 F F F
37042024-02-07T18:56:02.995Z 26 520-539 32 32 32 60 60 60 F F F
37052024-02-07T18:56:02.995Z 27 540-559 32 32 32 60 60 60 F F F
37062024-02-07T18:56:02.995Z 28 560-579 32 32 32 60 60 60 F F F
37072024-02-07T18:56:02.995Z 29 580-599 32 32 32 60 60 60 F F F
37082024-02-07T18:56:02.995ZMax gen: 33, Max flush: 61
37092024-02-07T18:56:02.995Z
37102024-02-07T18:56:02.995ZBegin loop 17
37112024-02-07T18:56:02.995ZDownstairs to restart: 0
37122024-02-07T18:56:03.005Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.265938793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271}
37132024-02-07T18:56:03.006Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266165202Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271}
37142024-02-07T18:56:03.006Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266201658Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271}
37152024-02-07T18:56:03.419Z{"msg":"Crucible d15cdb99-38b4-4aa0-82cb-e8aea46ae469 has session id: ebafd8d5-5907-48b2-9bd6-d800c7cf96c0","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266226254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37162024-02-07T18:56:03.419ZCrucible runtime is spawned
37172024-02-07T18:56:03.419Z{"msg":"Upstairs opts: Upstairs UUID: d15cdb99-38b4-4aa0-82cb-e8aea46ae469, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266283616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37182024-02-07T18:56:03.419Z{"msg":"Crucible stats registered with UUID: d15cdb99-38b4-4aa0-82cb-e8aea46ae469","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266322741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37192024-02-07T18:56:03.419Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266347298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271}
37202024-02-07T18:56:03.419Z{"msg":"d15cdb99-38b4-4aa0-82cb-e8aea46ae469 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266513105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37212024-02-07T18:56:03.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266563738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37222024-02-07T18:56:03.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266600953Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37232024-02-07T18:56:03.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266648047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37242024-02-07T18:56:03.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.26669733Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37252024-02-07T18:56:03.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266730706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37262024-02-07T18:56:03.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.26677433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37272024-02-07T18:56:03.419Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266804706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37282024-02-07T18:56:03.419Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266838721Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37292024-02-07T18:56:03.419Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.266872917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37302024-02-07T18:56:03.419Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:02.266909642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37312024-02-07T18:56:03.419Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:02.266946417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37322024-02-07T18:56:03.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.26699432Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37332024-02-07T18:56:03.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267027906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37342024-02-07T18:56:03.419Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:02.267082928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37352024-02-07T18:56:03.420Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267108675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37362024-02-07T18:56:03.420Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267126442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37372024-02-07T18:56:03.420Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267155529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37382024-02-07T18:56:03.420Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267604018Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37392024-02-07T18:56:03.420Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267643952Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37402024-02-07T18:56:03.420Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267705494Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37412024-02-07T18:56:03.420Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.267744829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37422024-02-07T18:56:03.420Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268210956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37432024-02-07T18:56:03.420Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268363485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37442024-02-07T18:56:03.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268748353Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37452024-02-07T18:56:03.420Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268785488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37462024-02-07T18:56:03.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268802106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37472024-02-07T18:56:03.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268821073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37482024-02-07T18:56:03.420Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.26884583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37492024-02-07T18:56:03.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:02.268872576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37502024-02-07T18:56:04.006Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.267972808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37512024-02-07T18:56:04.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.268289575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37522024-02-07T18:56:04.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.269404884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37532024-02-07T18:56:04.013Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.269470975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37542024-02-07T18:56:04.013Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.269982246Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37552024-02-07T18:56:04.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.27054594Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37562024-02-07T18:56:04.014Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270576576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37572024-02-07T18:56:04.014Z{"msg":"[0]R flush_numbers[0..12]: [57, 57, 56, 56, 59, 59, 50, 56, 51, 61, 60, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270592703Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37582024-02-07T18:56:04.014Z{"msg":"[0]R generation[0..12]: [31, 31, 30, 30, 32, 32, 26, 30, 27, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.27062027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37592024-02-07T18:56:04.014Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270635328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37602024-02-07T18:56:04.014Z{"msg":"[1]R flush_numbers[0..12]: [57, 57, 56, 56, 59, 59, 50, 56, 51, 61, 60, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270650945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37612024-02-07T18:56:04.014Z{"msg":"[1]R generation[0..12]: [31, 31, 30, 30, 32, 32, 26, 30, 27, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270673272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37622024-02-07T18:56:04.014Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.27068892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37632024-02-07T18:56:04.014Z{"msg":"[2]R flush_numbers[0..12]: [57, 57, 56, 56, 59, 59, 50, 56, 51, 61, 60, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270715577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37642024-02-07T18:56:04.014Z{"msg":"[2]R generation[0..12]: [31, 31, 30, 30, 32, 32, 26, 30, 27, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270730575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37652024-02-07T18:56:04.014Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
37662024-02-07T18:56:04.014Z,"time":"2024-02-07T18:56:03.270745033Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37672024-02-07T18:56:04.014Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270758511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37682024-02-07T18:56:04.014Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270772979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37692024-02-07T18:56:04.014Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270786387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37702024-02-07T18:56:04.014Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270799945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37712024-02-07T18:56:04.014Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270820792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37722024-02-07T18:56:04.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270833371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37732024-02-07T18:56:04.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270853638Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37742024-02-07T18:56:04.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270867346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37752024-02-07T18:56:04.014Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270881504Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37762024-02-07T18:56:04.014Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270907861Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37772024-02-07T18:56:04.014Z{"msg":"d15cdb99-38b4-4aa0-82cb-e8aea46ae469 is now active with session: ebafd8d5-5907-48b2-9bd6-d800c7cf96c0","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270922309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37782024-02-07T18:56:04.015Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270941586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271,"session_id":"ebafd8d5-5907-48b2-9bd6-d800c7cf96c0"}
37792024-02-07T18:56:04.015Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:03.270954264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271}
37802024-02-07T18:56:04.015ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37812024-02-07T18:56:04.015ZLoading write count information from file "/tmp/test_repair_verify.data"
37822024-02-07T18:56:04.015ZRun Repair workload
37832024-02-07T18:56:04.015Z01/30 Write block 109 len 20480 data: 5 4 4 3 3
37842024-02-07T18:56:04.015Z02/30 Write block 590 len 36864 data: 3 5 5 4 4 3 2 2 2
37852024-02-07T18:56:04.018Z03/30 Write block 295 len 4096 data: 4
37862024-02-07T18:56:04.019Z04/30 Read block 211 len 4096
37872024-02-07T18:56:04.019Z05/30 Read block 400 len 24576
37882024-02-07T18:56:04.019Z06/30 Flush
37892024-02-07T18:56:04.019Z07/30 Flush
37902024-02-07T18:56:04.019Z08/30 Write block 386 len 4096 data: 4
37912024-02-07T18:56:04.019Z09/30 Write block 342 len 28672 data: 3 3 4 6 7 5 5
37922024-02-07T18:56:04.019Z10/30 Write block 402 len 32768 data: 3 3 3 2 2 2 2 3
37932024-02-07T18:56:04.019Z11/30 Flush
37942024-02-07T18:56:04.023Z12/30 Write block 115 len 32768 data: 2 2 3 5 6 5 4 4
37952024-02-07T18:56:04.023Z13/30 Write block 461 len 12288 data: 2 2 3
37962024-02-07T18:56:04.026Z14/30 Read block 469 len 28672
37972024-02-07T18:56:04.026Z15/30 Read block 106 len 28672
37982024-02-07T18:56:04.026Z16/30 Read block 414 len 28672
37992024-02-07T18:56:04.026Z17/30 Write block 121 len 36864 data: 5 5 3 3 2 2 2 3 3
38002024-02-07T18:56:04.026Z18/30 Read block 344 len 28672
38012024-02-07T18:56:04.029Z19/30 Read block 551 len 4096
38022024-02-07T18:56:04.029Z20/30 Read block 438 len 40960
38032024-02-07T18:56:04.029Z21/30 Read block 500 len 36864
38042024-02-07T18:56:04.032Z22/30 Read block 261 len 12288
38052024-02-07T18:56:04.033Z23/30 Read block 245 len 28672
38062024-02-07T18:56:04.033Z24/30 Flush
38072024-02-07T18:56:04.036Z25/30 Write block 246 len 8192 data: 4 5
38082024-02-07T18:56:04.036Z26/30 Read block 328 len 12288
38092024-02-07T18:56:04.036Z27/30 Read block 387 len 24576
38102024-02-07T18:56:04.039Z28/30 Read block 421 len 20480
38112024-02-07T18:56:04.040Z29/30 Read block 327 len 24576
38122024-02-07T18:56:04.040Z30/30 Read block 351 len 28672
38132024-02-07T18:56:04.046Z----------------------------------------------------------------
38142024-02-07T18:56:04.046Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
38152024-02-07T18:56:04.046ZStates: Active Active Active
38162024-02-07T18:56:04.046ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
38172024-02-07T18:56:04.046Z 1 Acked 1000 Write 5 Sent Done Done false
38182024-02-07T18:56:04.046Z 2 Acked 1001 Write 9 Sent Done Done false
38192024-02-07T18:56:04.046Z 3 Acked 1002 Write 1 Sent Done Done false
38202024-02-07T18:56:04.046Z 4 Acked 1003 Read 1 Sent Done Done false
38212024-02-07T18:56:04.046Z 5 Acked 1004 Read 6 Sent Done Done false
38222024-02-07T18:56:04.047Z 6 Acked 1005 Flush 0 Sent Done Done false
38232024-02-07T18:56:04.047Z 7 Acked 1006 Flush 0 Sent Done Done false
38242024-02-07T18:56:04.047Z 8 Acked 1007 Write 1 Sent Done Done false
38252024-02-07T18:56:04.047Z 9 Acked 1008 Write 7 Sent Done Done false
38262024-02-07T18:56:04.047Z 10 Acked 1009 Write 8 Sent Done Done false
38272024-02-07T18:56:04.047Z 11 Acked 1010 Flush 0 Sent Done Done false
38282024-02-07T18:56:04.047Z 12 Acked 1011 Write 8 Sent Done Done false
38292024-02-07T18:56:04.047Z 13 Acked 1012 Write 3 Sent Done Done false
38302024-02-07T18:56:04.047Z 14 Acked 1013 Read 7 Sent Done Done false
38312024-02-07T18:56:04.047Z 15 Acked 1014 Read 7 Sent Done Done false
38322024-02-07T18:56:04.047Z 16 Acked 1015 Read 7 Sent Done Done false
38332024-02-07T18:56:04.047Z 17 Acked 1016 Write 9 Sent Done Done false
38342024-02-07T18:56:04.047Z 18 Acked 1017 Read 7 Sent Done Done false
38352024-02-07T18:56:04.047Z 19 Acked 1018 Read 1 Sent Done Done false
38362024-02-07T18:56:04.047Z 20 Acked 1019 Read 10 Sent Done Done false
38372024-02-07T18:56:04.047Z 21 Acked 1020 Read 9 Sent Done Done false
38382024-02-07T18:56:04.047Z 22 Acked 1021 Read 3 Sent Done Done false
38392024-02-07T18:56:04.047Z 23 Acked 1022 Read 7 Sent Done Done false
38402024-02-07T18:56:04.047Z 24 Acked 1023 Flush 0 Sent Done Done false
38412024-02-07T18:56:04.047Z 25 Acked 1024 Write 2 Sent Done Done false
38422024-02-07T18:56:04.047Z 26 Acked 1025 Read 3 Sent Done Done false
38432024-02-07T18:56:04.047Z 27 Acked 1026 Read 6 Sent Done Done false
38442024-02-07T18:56:04.047Z 28 Acked 1027 Read 5 Sent Done Done false
38452024-02-07T18:56:04.047Z 29 Acked 1028 Read 6 Sent Done Done false
38462024-02-07T18:56:04.048Z 30 Acked 1029 Read 7 Sent Done Done false
38472024-02-07T18:56:04.048Z STATES DS:0 DS:1 DS:2 TOTAL
38482024-02-07T18:56:04.048Z New 0 0 0 0
38492024-02-07T18:56:04.048Z Sent 30 0 0 30
38502024-02-07T18:56:04.048Z Done 0 30 30 60
38512024-02-07T18:56:04.048Z Skipped 0 0 0 0
38522024-02-07T18:56:04.048Z Error 0 0 0 0
38532024-02-07T18:56:04.048ZLast Flush: 0 1023 1023
38542024-02-07T18:56:04.048ZDownstairs last five completed:
38552024-02-07T18:56:04.048ZUpstairs last five completed: 30 29 28 27 26
38562024-02-07T18:56:04.048Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:03.302952327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1271}
38572024-02-07T18:56:04.048ZWrote out file "/tmp/test_repair_verify.data"
38582024-02-07T18:56:04.051Z
38592024-02-07T18:56:04.051ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38602024-02-07T18:56:04.064ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38612024-02-07T18:56:04.064Z 0 000-019 31 31 31 57 57 57 F F F
38622024-02-07T18:56:04.064Z 1 020-039 31 31 31 57 57 57 F F F
38632024-02-07T18:56:04.064Z 2 040-059 30 30 30 56 56 56 F F F
38642024-02-07T18:56:04.064Z 3 060-079 30 30 30 56 56 56 F F F
38652024-02-07T18:56:04.064Z 4 080-099 32 32 32 59 59 59 F F F
38662024-02-07T18:56:04.064Z 5 100-119 32 34 34 59 65 65 F F F <---
38672024-02-07T18:56:04.064Z 6 120-139 26 34 34 50 65 65 F F F <---
38682024-02-07T18:56:04.064Z 7 140-159 30 30 30 56 56 56 F F F
38692024-02-07T18:56:04.065Z 8 160-179 27 27 27 51 51 51 F F F
38702024-02-07T18:56:04.065Z 9 180-199 33 33 33 61 61 61 F F F
38712024-02-07T18:56:04.065Z 10 200-219 32 32 32 60 60 60 F F F
38722024-02-07T18:56:04.065Z 11 220-239 31 31 31 57 57 57 F F F
38732024-02-07T18:56:04.065Z 12 240-259 33 33 33 61 61 61 F T T <---
38742024-02-07T18:56:04.069Z 13 260-279 33 33 33 61 61 61 F F F
38752024-02-07T18:56:04.069Z 14 280-299 33 34 34 61 62 62 F F F <---
38762024-02-07T18:56:04.069Z 15 300-319 33 33 33 61 61 61 F F F
38772024-02-07T18:56:04.069Z 16 320-339 24 24 24 45 45 45 F F F
38782024-02-07T18:56:04.069Z 17 340-359 32 34 34 60 64 64 F F F <---
38792024-02-07T18:56:04.069Z 18 360-379 33 33 33 61 61 61 F F F
38802024-02-07T18:56:04.069Z 19 380-399 32 34 34 58 64 64 F F F <---
38812024-02-07T18:56:04.069Z 20 400-419 28 34 34 53 64 64 F F F <---
38822024-02-07T18:56:04.069Z 21 420-439 30 30 30 56 56 56 F F F
38832024-02-07T18:56:04.069Z 22 440-459 28 28 28 53 53 53 F F F
38842024-02-07T18:56:04.069Z 23 460-479 27 34 34 51 65 65 F F F <---
38852024-02-07T18:56:04.069Z 24 480-499 31 31 31 57 57 57 F F F
38862024-02-07T18:56:04.069Z 25 500-519 32 32 32 59 59 59 F F F
38872024-02-07T18:56:04.069Z 26 520-539 32 32 32 60 60 60 F F F
38882024-02-07T18:56:04.069Z 27 540-559 32 32 32 60 60 60 F F F
38892024-02-07T18:56:04.069Z 28 560-579 32 32 32 60 60 60 F F F
38902024-02-07T18:56:04.069Z 29 580-599 32 34 34 60 62 62 F F F <---
38912024-02-07T18:56:04.069ZMax gen: 34, Max flush: 65
38922024-02-07T18:56:04.069ZError: Difference in extent metadata found!
38932024-02-07T18:56:04.069ZA Difference in extent metadata is expected here
38942024-02-07T18:56:04.069ZOn loop 17
38952024-02-07T18:56:04.070Z
38962024-02-07T18:56:04.073ZVerifying data now
38972024-02-07T18:56:08.534ZNo change after verify
38982024-02-07T18:56:08.534ZLoop: 17 Downstairs dump after verify (and repair):
38992024-02-07T18:56:08.553ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39002024-02-07T18:56:08.553Z 0 000-019 31 31 31 57 57 57 F F F
39012024-02-07T18:56:08.553Z 1 020-039 31 31 31 57 57 57 F F F
39022024-02-07T18:56:08.553Z 2 040-059 30 30 30 56 56 56 F F F
39032024-02-07T18:56:08.553Z 3 060-079 30 30 30 56 56 56 F F F
39042024-02-07T18:56:08.553Z 4 080-099 32 32 32 59 59 59 F F F
39052024-02-07T18:56:08.564Z 5 100-119 34 34 34 65 65 65 F F F
39062024-02-07T18:56:08.564Z 6 120-139 34 34 34 65 65 65 F F F
39072024-02-07T18:56:08.564Z 7 140-159 30 30 30 56 56 56 F F F
39082024-02-07T18:56:08.564Z 8 160-179 27 27 27 51 51 51 F F F
39092024-02-07T18:56:08.564Z 9 180-199 33 33 33 61 61 61 F F F
39102024-02-07T18:56:08.564Z 10 200-219 32 32 32 60 60 60 F F F
39112024-02-07T18:56:08.565Z 11 220-239 31 31 31 57 57 57 F F F
39122024-02-07T18:56:08.565Z 12 240-259 35 35 35 66 66 66 F F F
39132024-02-07T18:56:08.565Z 13 260-279 33 33 33 61 61 61 F F F
39142024-02-07T18:56:08.565Z 14 280-299 34 34 34 62 62 62 F F F
39152024-02-07T18:56:08.565Z 15 300-319 33 33 33 61 61 61 F F F
39162024-02-07T18:56:08.565Z 16 320-339 24 24 24 45 45 45 F F F
39172024-02-07T18:56:08.565Z 17 340-359 34 34 34 64 64 64 F F F
39182024-02-07T18:56:08.565Z 18 360-379 33 33 33 61 61 61 F F F
39192024-02-07T18:56:08.565Z 19 380-399 34 34 34 64 64 64 F F F
39202024-02-07T18:56:08.565Z 20 400-419 34 34 34 64 64 64 F F F
39212024-02-07T18:56:08.565Z 21 420-439 30 30 30 56 56 56 F F F
39222024-02-07T18:56:08.565Z 22 440-459 28 28 28 53 53 53 F F F
39232024-02-07T18:56:08.565Z 23 460-479 34 34 34 65 65 65 F F F
39242024-02-07T18:56:08.565Z 24 480-499 31 31 31 57 57 57 F F F
39252024-02-07T18:56:08.565Z 25 500-519 32 32 32 59 59 59 F F F
39262024-02-07T18:56:08.565Z 26 520-539 32 32 32 60 60 60 F F F
39272024-02-07T18:56:08.565Z 27 540-559 32 32 32 60 60 60 F F F
39282024-02-07T18:56:08.565Z 28 560-579 32 32 32 60 60 60 F F F
39292024-02-07T18:56:08.565Z 29 580-599 34 34 34 62 62 62 F F F
39302024-02-07T18:56:08.565ZMax gen: 35, Max flush: 66
39312024-02-07T18:56:08.565Z
39322024-02-07T18:56:08.565ZBegin loop 18
39332024-02-07T18:56:08.565ZDownstairs to restart: 1
39342024-02-07T18:56:08.576Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834209511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279}
39352024-02-07T18:56:08.576Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834485644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279}
39362024-02-07T18:56:08.613Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
39372024-02-07T18:56:08.613Z,"time":"2024-02-07T18:56:07.834530398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279}
39382024-02-07T18:56:08.613Z{"msg":"Crucible ebb3b268-e677-40b7-b12b-ce86d9c35094 has session id: 38d90a7b-f890-45cb-8a59-5cd3ea73c313","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834599749Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39392024-02-07T18:56:08.613Z{"msg":"Upstairs opts: Upstairs UUID: ebb3b268-e677-40b7-b12b-ce86d9c35094, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834652131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39402024-02-07T18:56:08.613Z{"msg":"Crucible stats registered with UUID: ebb3b268-e677-40b7-b12b-ce86d9c35094","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834674648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39412024-02-07T18:56:08.613Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834688417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279}
39422024-02-07T18:56:08.614Z{"msg":"ebb3b268-e677-40b7-b12b-ce86d9c35094 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834831147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39432024-02-07T18:56:08.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834867652Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39442024-02-07T18:56:08.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834896018Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39452024-02-07T18:56:08.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834921885Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39462024-02-07T18:56:08.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.834967779Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39472024-02-07T18:56:08.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835004764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39482024-02-07T18:56:08.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.83503172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39492024-02-07T18:56:08.614Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835059406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39502024-02-07T18:56:08.614Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835087852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39512024-02-07T18:56:08.614Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835114239Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39522024-02-07T18:56:08.614Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835140615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39532024-02-07T18:56:08.615Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835169241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39542024-02-07T18:56:08.615Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835390391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39552024-02-07T18:56:08.615Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835869637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39562024-02-07T18:56:08.615Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.83591843Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39572024-02-07T18:56:08.615Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835948746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39582024-02-07T18:56:08.615Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.835975392Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39592024-02-07T18:56:08.615Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.836485454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39602024-02-07T18:56:08.615Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.836639533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39612024-02-07T18:56:08.615Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837007123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39622024-02-07T18:56:08.615Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837042928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39632024-02-07T18:56:08.615Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837060746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39642024-02-07T18:56:08.615Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837092591Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39652024-02-07T18:56:08.616Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837113569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39662024-02-07T18:56:08.616Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837239332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39672024-02-07T18:56:08.616Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837268757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39682024-02-07T18:56:08.616Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837302963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39692024-02-07T18:56:08.616Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.837648756Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39702024-02-07T18:56:08.616Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838104824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39712024-02-07T18:56:08.616Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838153818Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39722024-02-07T18:56:08.616Z{"msg":"[0]R flush_numbers[0..12]: [57, 57, 56, 56, 59, 65, 65, 56, 51, 61, 60, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838173135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39732024-02-07T18:56:08.617Z{"msg":"[0]R generation[0..12]: [31, 31, 30, 30, 32, 34, 34, 30, 27, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838188013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39742024-02-07T18:56:08.617Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.83821059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39752024-02-07T18:56:08.617Z{"msg":"[1]R flush_numbers[0..12]: [57, 57, 56, 56, 59, 65, 65, 56, 51, 61, 60, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838225678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39762024-02-07T18:56:08.617Z{"msg":"[1]R generation[0..12]: [31, 31, 30, 30, 32, 34, 34, 30, 27, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838240016Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39772024-02-07T18:56:08.617Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838254254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39782024-02-07T18:56:08.617Z{"msg":"[2]R flush_numbers[0..12]: [57, 57, 56, 56, 59, 65, 65, 56, 51, 61, 60, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838268652Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39792024-02-07T18:56:08.617Z{"msg":"[2]R generation[0..12]: [31, 31, 30, 30, 32, 34, 34, 30, 27, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838290059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39802024-02-07T18:56:08.617Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
39812024-02-07T18:56:08.617Z,"time":"2024-02-07T18:56:07.838304957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39822024-02-07T18:56:08.618Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838319735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39832024-02-07T18:56:08.618Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838333804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39842024-02-07T18:56:08.618Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838346922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39852024-02-07T18:56:08.618Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838367139Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39862024-02-07T18:56:08.618Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838380887Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39872024-02-07T18:56:08.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838393036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39882024-02-07T18:56:08.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838413423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39892024-02-07T18:56:08.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838427021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39902024-02-07T18:56:08.619Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838441149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39912024-02-07T18:56:08.619Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838460246Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39922024-02-07T18:56:08.619Z{"msg":"ebb3b268-e677-40b7-b12b-ce86d9c35094 is now active with session: 38d90a7b-f890-45cb-8a59-5cd3ea73c313","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838473774Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39932024-02-07T18:56:08.619Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.838486443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279,"session_id":"38d90a7b-f890-45cb-8a59-5cd3ea73c313"}
39942024-02-07T18:56:08.619Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:07.83850549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279}
39952024-02-07T18:56:08.619ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39962024-02-07T18:56:08.619ZLoading write count information from file "/tmp/test_repair_verify.data"
39972024-02-07T18:56:08.619ZRun Repair workload
39982024-02-07T18:56:08.620Z01/30 Write block 548 len 8192 data: 3 4
39992024-02-07T18:56:08.620Z02/30 Write block 436 len 36864 data: 5 6 4 5 6 5 7 7 6
40002024-02-07T18:56:08.620Z03/30 Read block 14 len 36864
40012024-02-07T18:56:08.636Z04/30 Write block 18 len 12288 data: 5 6 6
40022024-02-07T18:56:08.636Z05/30 Write block 9 len 20480 data: 3 3 3 2 3
40032024-02-07T18:56:08.636Z06/30 Write block 450 len 12288 data: 3 3 3
40042024-02-07T18:56:08.636Z07/30 Flush
40052024-02-07T18:56:08.637Z08/30 Write block 208 len 28672 data: 4 2 2 2 2 3 3
40062024-02-07T18:56:08.637Z09/30 Read block 55 len 20480
40072024-02-07T18:56:08.637Z10/30 Flush
40082024-02-07T18:56:08.637Z11/30 Write block 22 len 40960 data: 6 7 7 6 4 2 3 4 4 4
40092024-02-07T18:56:08.637Z12/30 Read block 404 len 4096
40102024-02-07T18:56:08.637Z13/30 Write block 393 len 4096 data: 6
40112024-02-07T18:56:08.637Z14/30 Read block 567 len 16384
40122024-02-07T18:56:08.638Z15/30 Read block 542 len 36864
40132024-02-07T18:56:08.638Z16/30 Read block 83 len 16384
40142024-02-07T18:56:08.638Z17/30 Read block 523 len 24576
40152024-02-07T18:56:08.638Z18/30 Read block 553 len 4096
40162024-02-07T18:56:08.638Z19/30 Flush
40172024-02-07T18:56:08.638Z20/30 Write block 146 len 36864 data: 4 4 5 5 4 4 4 4 3
40182024-02-07T18:56:08.638Z21/30 Write block 453 len 8192 data: 4 3
40192024-02-07T18:56:08.639Z22/30 Read block 48 len 8192
40202024-02-07T18:56:08.639Z23/30 Write block 259 len 32768 data: 5 5 5 5 5 5 4 2
40212024-02-07T18:56:08.639Z24/30 Write block 146 len 36864 data: 5 5 6 6 5 5 5 5 4
40222024-02-07T18:56:08.639Z25/30 Read block 530 len 16384
40232024-02-07T18:56:08.639Z26/30 Read block 508 len 16384
40242024-02-07T18:56:08.639Z27/30 Write block 106 len 16384 data: 4 4 4 6
40252024-02-07T18:56:08.640Z28/30 Read block 87 len 32768
40262024-02-07T18:56:08.640Z29/30 Write block 235 len 8192 data: 8 6
40272024-02-07T18:56:08.640Z30/30 Read block 188 len 24576
40282024-02-07T18:56:08.640Z----------------------------------------------------------------
40292024-02-07T18:56:08.640Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
40302024-02-07T18:56:08.640ZStates: Active Active Active
40312024-02-07T18:56:08.640ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40322024-02-07T18:56:08.641Z 1 Acked 1000 Write 2 Done Done Done false
40332024-02-07T18:56:08.641Z 2 Acked 1001 Write 9 Done Sent Done false
40342024-02-07T18:56:08.641Z 3 Acked 1002 Read 9 Done Sent Done false
40352024-02-07T18:56:08.641Z 4 Acked 1003 Write 3 Done Sent Done false
40362024-02-07T18:56:08.641Z 5 Acked 1004 Write 5 Done Sent Done false
40372024-02-07T18:56:08.641Z 6 Acked 1005 Write 3 Done Sent Done false
40382024-02-07T18:56:08.641Z 7 Acked 1006 Flush 0 Done Sent Done false
40392024-02-07T18:56:08.642Z 8 Acked 1007 Write 7 Done Sent Done false
40402024-02-07T18:56:08.642Z 9 Acked 1008 Read 5 Done Sent Done false
40412024-02-07T18:56:08.642Z 10 Acked 1009 Flush 0 Done Sent Done false
40422024-02-07T18:56:08.642Z 11 Acked 1010 Write 10 Done Sent Done false
40432024-02-07T18:56:08.642Z 12 Acked 1011 Read 1 Done Sent Done false
40442024-02-07T18:56:08.642Z 13 Acked 1012 Write 1 Done Sent Done false
40452024-02-07T18:56:08.643Z 14 Acked 1013 Read 4 Done Sent Done false
40462024-02-07T18:56:08.643Z 15 Acked 1014 Read 9 Done Sent Done false
40472024-02-07T18:56:08.643Z 16 Acked 1015 Read 4 Done Sent Done false
40482024-02-07T18:56:08.643Z 17 Acked 1016 Read 6 Done Sent Done false
40492024-02-07T18:56:08.643Z 18 Acked 1017 Read 1 Done Sent Done false
40502024-02-07T18:56:08.643Z 19 Acked 1018 Flush 0 Done Sent Done false
40512024-02-07T18:56:08.643Z 20 Acked 1019 Write 9 Done Sent Done false
40522024-02-07T18:56:08.644Z 21 Acked 1020 Write 2 Done Sent Done false
40532024-02-07T18:56:08.644Z 22 Acked 1021 Read 2 Done Sent Done false
40542024-02-07T18:56:08.644Z 23 Acked 1022 Write 8 Done Sent Done false
40552024-02-07T18:56:08.644Z 24 Acked 1023 Write 9 Done Sent Done false
40562024-02-07T18:56:08.644Z 25 Acked 1024 Read 4 Done Sent Done false
40572024-02-07T18:56:08.644Z 26 Acked 1025 Read 4 Done Sent Done false
40582024-02-07T18:56:08.645Z 27 Acked 1026 Write 4 Done Sent Done false
40592024-02-07T18:56:08.645Z 28 Acked 1027 Read 8 Done Sent Done false
40602024-02-07T18:56:08.645Z 29 Acked 1028 Write 2 Done Sent Done false
40612024-02-07T18:56:08.645Z 30 Acked 1029 Read 6 Done Sent Sent false
40622024-02-07T18:56:08.645Z STATES DS:0 DS:1 DS:2 TOTAL
40632024-02-07T18:56:08.645Z New 0 0 0 0
40642024-02-07T18:56:08.645Z Sent 0 29 1 30
40652024-02-07T18:56:08.646Z Done 30 1 29 60
40662024-02-07T18:56:08.646Z Skipped 0 0 0 0
40672024-02-07T18:56:08.646Z Error 0 0 0 0
40682024-02-07T18:56:08.646ZLast Flush: 1018 0 1018
40692024-02-07T18:56:08.646ZDownstairs last five completed:
40702024-02-07T18:56:08.646ZUpstairs last five completed: 30 29 28 27 26
40712024-02-07T18:56:08.646Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:07.87007589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1279}
40722024-02-07T18:56:08.646ZWrote out file "/tmp/test_repair_verify.data"
40732024-02-07T18:56:08.647Z
40742024-02-07T18:56:08.647ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40752024-02-07T18:56:08.654ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40762024-02-07T18:56:08.654Z 0 000-019 36 31 36 67 57 67 F F F <---
40772024-02-07T18:56:08.654Z 1 020-039 36 31 36 69 57 69 F F F <---
40782024-02-07T18:56:08.654Z 2 040-059 30 30 30 56 56 56 F F F
40792024-02-07T18:56:08.654Z 3 060-079 30 30 30 56 56 56 F F F
40802024-02-07T18:56:08.654Z 4 080-099 32 32 32 59 59 59 F F F
40812024-02-07T18:56:08.654Z 5 100-119 34 34 34 65 65 65 T F T <---
40822024-02-07T18:56:08.654Z 6 120-139 34 34 34 65 65 65 F F F
40832024-02-07T18:56:08.654Z 7 140-159 30 30 30 56 56 56 T F T <---
40842024-02-07T18:56:08.654Z 8 160-179 27 27 27 51 51 51 F F F
40852024-02-07T18:56:08.654Z 9 180-199 33 33 33 61 61 61 F F F
40862024-02-07T18:56:08.654Z 10 200-219 36 32 36 68 60 68 F F F <---
40872024-02-07T18:56:08.654Z 11 220-239 31 31 31 57 57 57 T F T <---
40882024-02-07T18:56:08.654Z 12 240-259 35 35 35 66 66 66 T F T <---
40892024-02-07T18:56:08.654Z 13 260-279 33 33 33 61 61 61 T F T <---
40902024-02-07T18:56:08.654Z 14 280-299 34 34 34 62 62 62 F F F
40912024-02-07T18:56:08.655Z 15 300-319 33 33 33 61 61 61 F F F
40922024-02-07T18:56:08.655Z 16 320-339 24 24 24 45 45 45 F F F
40932024-02-07T18:56:08.655Z 17 340-359 34 34 34 64 64 64 F F F
40942024-02-07T18:56:08.655Z 18 360-379 33 33 33 61 61 61 F F F
40952024-02-07T18:56:08.655Z 19 380-399 36 34 36 69 64 69 F F F <---
40962024-02-07T18:56:08.655Z 20 400-419 34 34 34 64 64 64 F F F
40972024-02-07T18:56:08.655Z 21 420-439 36 30 36 67 56 67 F F F <---
40982024-02-07T18:56:08.655Z 22 440-459 36 28 36 67 53 67 T F T <---
40992024-02-07T18:56:08.655Z 23 460-479 34 34 34 65 65 65 F F F
41002024-02-07T18:56:08.655Z 24 480-499 31 31 31 57 57 57 F F F
41012024-02-07T18:56:08.655Z 25 500-519 32 32 32 59 59 59 F F F
41022024-02-07T18:56:08.655Z 26 520-539 32 32 32 60 60 60 F F F
41032024-02-07T18:56:08.655Z 27 540-559 36 32 36 67 60 67 F T F <---
41042024-02-07T18:56:08.655Z 28 560-579 32 32 32 60 60 60 F F F
41052024-02-07T18:56:08.655Z 29 580-599 34 34 34 62 62 62 F F F
41062024-02-07T18:56:08.655ZMax gen: 36, Max flush: 69
41072024-02-07T18:56:08.655ZError: Difference in extent metadata found!
41082024-02-07T18:56:08.655ZA Difference in extent metadata is expected here
41092024-02-07T18:56:08.655ZOn loop 18
41102024-02-07T18:56:08.655Z
41112024-02-07T18:56:08.655ZVerifying data now
41122024-02-07T18:56:13.245ZNo change after verify
41132024-02-07T18:56:13.248ZLoop: 18 Downstairs dump after verify (and repair):
41142024-02-07T18:56:13.261ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41152024-02-07T18:56:13.261Z 0 000-019 36 36 36 67 67 67 F F F
41162024-02-07T18:56:13.261Z 1 020-039 36 36 36 69 69 69 F F F
41172024-02-07T18:56:13.261Z 2 040-059 30 30 30 56 56 56 F F F
41182024-02-07T18:56:13.261Z 3 060-079 30 30 30 56 56 56 F F F
41192024-02-07T18:56:13.261Z 4 080-099 32 32 32 59 59 59 F F F
41202024-02-07T18:56:13.261Z 5 100-119 37 37 37 70 70 70 F F F
41212024-02-07T18:56:13.261Z 6 120-139 34 34 34 65 65 65 F F F
41222024-02-07T18:56:13.261Z 7 140-159 37 37 37 70 70 70 F F F
41232024-02-07T18:56:13.261Z 8 160-179 27 27 27 51 51 51 F F F
41242024-02-07T18:56:13.261Z 9 180-199 33 33 33 61 61 61 F F F
41252024-02-07T18:56:13.261Z 10 200-219 36 36 36 68 68 68 F F F
41262024-02-07T18:56:13.261Z 11 220-239 37 37 37 70 70 70 F F F
41272024-02-07T18:56:13.265Z 12 240-259 37 37 37 70 70 70 F F F
41282024-02-07T18:56:13.265Z 13 260-279 37 37 37 70 70 70 F F F
41292024-02-07T18:56:13.265Z 14 280-299 34 34 34 62 62 62 F F F
41302024-02-07T18:56:13.265Z 15 300-319 33 33 33 61 61 61 F F F
41312024-02-07T18:56:13.265Z 16 320-339 24 24 24 45 45 45 F F F
41322024-02-07T18:56:13.265Z 17 340-359 34 34 34 64 64 64 F F F
41332024-02-07T18:56:13.265Z 18 360-379 33 33 33 61 61 61 F F F
41342024-02-07T18:56:13.265Z 19 380-399 36 36 36 69 69 69 F F F
41352024-02-07T18:56:13.265Z 20 400-419 34 34 34 64 64 64 F F F
41362024-02-07T18:56:13.265Z 21 420-439 36 36 36 67 67 67 F F F
41372024-02-07T18:56:13.265Z 22 440-459 37 37 37 70 70 70 F F F
41382024-02-07T18:56:13.265Z 23 460-479 34 34 34 65 65 65 F F F
41392024-02-07T18:56:13.265Z 24 480-499 31 31 31 57 57 57 F F F
41402024-02-07T18:56:13.265Z 25 500-519 32 32 32 59 59 59 F F F
41412024-02-07T18:56:13.265Z 26 520-539 32 32 32 60 60 60 F F F
41422024-02-07T18:56:13.265Z 27 540-559 36 36 36 67 67 67 F F F
41432024-02-07T18:56:13.265Z 28 560-579 32 32 32 60 60 60 F F F
41442024-02-07T18:56:13.265Z 29 580-599 34 34 34 62 62 62 F F F
41452024-02-07T18:56:13.265ZMax gen: 37, Max flush: 70
41462024-02-07T18:56:13.265Z
41472024-02-07T18:56:13.266ZBegin loop 19
41482024-02-07T18:56:13.266ZDownstairs to restart: 1
41492024-02-07T18:56:13.282Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.542652591Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287}
41502024-02-07T18:56:13.282Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
41512024-02-07T18:56:13.282Z,"time":"2024-02-07T18:56:12.54295451Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287}
41522024-02-07T18:56:13.286Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.542993955Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287}
41532024-02-07T18:56:13.286Z{"msg":"Crucible f0f4dc87-f6e2-4082-b68a-c7415ea80ceb has session id: 6ffc76d5-b2ec-4c94-9259-aebb9245be9c","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.54302878Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41542024-02-07T18:56:13.287Z{"msg":"Upstairs opts: Upstairs UUID: f0f4dc87-f6e2-4082-b68a-c7415ea80ceb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543078754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41552024-02-07T18:56:13.287Z{"msg":"Crucible stats registered with UUID: f0f4dc87-f6e2-4082-b68a-c7415ea80ceb","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543097391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41562024-02-07T18:56:13.287Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543128727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287}
41572024-02-07T18:56:13.287Z{"msg":"f0f4dc87-f6e2-4082-b68a-c7415ea80ceb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543142815Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41582024-02-07T18:56:13.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543173191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41592024-02-07T18:56:13.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543197897Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41602024-02-07T18:56:13.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543240392Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41612024-02-07T18:56:13.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543268158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41622024-02-07T18:56:13.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543292375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41632024-02-07T18:56:13.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543391321Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41642024-02-07T18:56:13.287Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543431446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41652024-02-07T18:56:13.287Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543462042Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41662024-02-07T18:56:13.287Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543502336Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41672024-02-07T18:56:13.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543530153Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41682024-02-07T18:56:13.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543556329Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41692024-02-07T18:56:13.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.543581356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41702024-02-07T18:56:13.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.544296699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41712024-02-07T18:56:13.287Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.544357371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41722024-02-07T18:56:13.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.544409964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41732024-02-07T18:56:13.287Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.54443477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41742024-02-07T18:56:13.287Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.544923804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41752024-02-07T18:56:13.287Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.544981216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41762024-02-07T18:56:13.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545386272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41772024-02-07T18:56:13.287Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545434475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41782024-02-07T18:56:13.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545591794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41792024-02-07T18:56:13.288Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545629169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41802024-02-07T18:56:13.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545645757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41812024-02-07T18:56:13.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545659795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41822024-02-07T18:56:13.288Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545674133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41832024-02-07T18:56:13.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545687801Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41842024-02-07T18:56:13.288Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.545943236Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41852024-02-07T18:56:13.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546400924Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41862024-02-07T18:56:13.292Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.54643465Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41872024-02-07T18:56:13.292Z{"msg":"[0]R flush_numbers[0..12]: [67, 69, 56, 56, 59, 70, 65, 70, 51, 61, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546450518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41882024-02-07T18:56:13.292Z{"msg":"[0]R generation[0..12]: [36, 36, 30, 30, 32, 37, 34, 37, 27, 33, 36, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546465496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41892024-02-07T18:56:13.292Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546552714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41902024-02-07T18:56:13.293Z{"msg":"[1]R flush_numbers[0..12]: [67, 69, 56, 56, 59, 70, 65, 70, 51, 61, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546589579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41912024-02-07T18:56:13.293Z{"msg":"[1]R generation[0..12]: [36, 36, 30, 30, 32, 37, 34, 37, 27, 33, 36, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546611606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41922024-02-07T18:56:13.293Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546642712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287Wait for a query_work_queue command to finish before sending IO
41932024-02-07T18:56:13.293Z,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41942024-02-07T18:56:13.293Z{"msg":"[2]R flush_numbers[0..12]: [67, 69, 56, 56, 59, 70, 65, 70, 51, 61, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546663149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41952024-02-07T18:56:13.293Z{"msg":"[2]R generation[0..12]: [36, 36, 30, 30, 32, 37, 34, 37, 27, 33, 36, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546695355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41962024-02-07T18:56:13.293Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546715222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41972024-02-07T18:56:13.293Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546744638Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41982024-02-07T18:56:13.293Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546762635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
41992024-02-07T18:56:13.293Z{"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546792142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42002024-02-07T18:56:13.293Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546806749Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42012024-02-07T18:56:13.293Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546837935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42022024-02-07T18:56:13.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546852433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42032024-02-07T18:56:13.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546883929Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42042024-02-07T18:56:13.293Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546903576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42052024-02-07T18:56:13.293Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546932972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42062024-02-07T18:56:13.293Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.54694719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42072024-02-07T18:56:13.293Z{"msg":"f0f4dc87-f6e2-4082-b68a-c7415ea80ceb is now active with session: 6ffc76d5-b2ec-4c94-9259-aebb9245be9c","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546961359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42082024-02-07T18:56:13.293Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.546990645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287,"session_id":"6ffc76d5-b2ec-4c94-9259-aebb9245be9c"}
42092024-02-07T18:56:13.293Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:12.547005173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287}
42102024-02-07T18:56:13.293ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
42112024-02-07T18:56:13.293ZLoading write count information from file "/tmp/test_repair_verify.data"
42122024-02-07T18:56:13.293ZRun Repair workload
42132024-02-07T18:56:13.293Z01/30 Write block 240 len 20480 data: 5 5 6 7 6
42142024-02-07T18:56:13.294Z02/30 Read block 198 len 16384
42152024-02-07T18:56:13.294Z03/30 Read block 114 len 4096
42162024-02-07T18:56:13.294Z04/30 Read block 302 len 12288
42172024-02-07T18:56:13.294Z05/30 Read block 316 len 20480
42182024-02-07T18:56:13.297Z06/30 Read block 76 len 32768
42192024-02-07T18:56:13.297Z07/30 Read block 258 len 8192
42202024-02-07T18:56:13.297Z08/30 Read block 90 len 28672
42212024-02-07T18:56:13.297Z09/30 Read block 23 len 36864
42222024-02-07T18:56:13.297Z10/30 Read block 364 len 16384
42232024-02-07T18:56:13.297Z11/30 Read block 564 len 40960
42242024-02-07T18:56:13.297Z12/30 Write block 66 len 32768 data: 6 6 6 7 6 6 4 4
42252024-02-07T18:56:13.300Z13/30 Write block 361 len 20480 data: 5 6 5 4 3
42262024-02-07T18:56:13.300Z14/30 Read block 346 len 12288
42272024-02-07T18:56:13.300Z15/30 Write block 386 len 16384 data: 5 4 5 6
42282024-02-07T18:56:13.300Z16/30 Write block 205 len 20480 data: 5 4 3 5 3
42292024-02-07T18:56:13.303Z17/30 Flush
42302024-02-07T18:56:13.306Z18/30 Write block 259 len 16384 data: 6 6 6 6
42312024-02-07T18:56:13.306Z19/30 Read block 517 len 32768
42322024-02-07T18:56:13.306Z20/30 Flush
42332024-02-07T18:56:13.309Z21/30 Write block 332 len 32768 data: 3 3 3 4 4 4 5 7
42342024-02-07T18:56:13.309Z22/30 Read block 118 len 36864
42352024-02-07T18:56:13.312Z23/30 Write block 535 len 40960 data: 2 2 3 3 3 3 4 5 2 2
42362024-02-07T18:56:13.312Z24/30 Read block 278 len 8192
42372024-02-07T18:56:13.312Z25/30 Write block 527 len 40960 data: 3 2 2 2 2 3 3 3 3 3
42382024-02-07T18:56:13.312Z26/30 Read block 330 len 24576
42392024-02-07T18:56:13.315Z27/30 Read block 506 len 20480
42402024-02-07T18:56:13.315Z28/30 Read block 253 len 4096
42412024-02-07T18:56:13.315Z29/30 Write block 318 len 36864 data: 5 4 5 4 4 4 3 2 2
42422024-02-07T18:56:13.319Z30/30 Read block 187 len 12288
42432024-02-07T18:56:13.319Z----------------------------------------------------------------
42442024-02-07T18:56:13.319Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42452024-02-07T18:56:13.319ZStates: Active Active Active
42462024-02-07T18:56:13.320ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42472024-02-07T18:56:13.320Z 1 Acked 1000 Write 5 Done Sent Done false
42482024-02-07T18:56:13.320Z 2 Acked 1001 Read 4 Done Sent Done false
42492024-02-07T18:56:13.320Z 3 Acked 1002 Read 1 Done Sent Done false
42502024-02-07T18:56:13.320Z 4 Acked 1003 Read 3 Done Sent Done false
42512024-02-07T18:56:13.320Z 5 Acked 1004 Read 5 Done Sent Done false
42522024-02-07T18:56:13.320Z 6 Acked 1005 Read 8 Done Sent Done false
42532024-02-07T18:56:13.320Z 7 Acked 1006 Read 2 Done Sent Done false
42542024-02-07T18:56:13.320Z 8 Acked 1007 Read 7 Done Sent Done false
42552024-02-07T18:56:13.320Z 9 Acked 1008 Read 9 Done Sent Done false
42562024-02-07T18:56:13.320Z 10 Acked 1009 Read 4 Done Sent Done false
42572024-02-07T18:56:13.320Z 11 Acked 1010 Read 10 Done Sent Done false
42582024-02-07T18:56:13.320Z 12 Acked 1011 Write 8 Done Sent Done false
42592024-02-07T18:56:13.320Z 13 Acked 1012 Write 5 Done Sent Done false
42602024-02-07T18:56:13.320Z 14 Acked 1013 Read 3 Done Sent Done false
42612024-02-07T18:56:13.320Z 15 Acked 1014 Write 4 Done Sent Done false
42622024-02-07T18:56:13.320Z 16 Acked 1015 Write 5 Done Sent Done false
42632024-02-07T18:56:13.320Z 17 Acked 1016 Flush 0 Done Sent Done false
42642024-02-07T18:56:13.320Z 18 Acked 1017 Write 4 Done Sent Done false
42652024-02-07T18:56:13.320Z 19 Acked 1018 Read 8 Done Sent Done false
42662024-02-07T18:56:13.320Z 20 Acked 1019 Flush 0 Done Sent Done false
42672024-02-07T18:56:13.320Z 21 Acked 1020 Write 8 Done Sent Done false
42682024-02-07T18:56:13.320Z 22 Acked 1021 Read 9 Done Sent Done false
42692024-02-07T18:56:13.320Z 23 Acked 1022 Write 10 Done Sent Done false
42702024-02-07T18:56:13.320Z 24 Acked 1023 Read 2 Done Sent Done false
42712024-02-07T18:56:13.321Z 25 Acked 1024 Write 10 Done Sent Done false
42722024-02-07T18:56:13.321Z 26 Acked 1025 Read 6 Done Sent Done false
42732024-02-07T18:56:13.321Z 27 Acked 1026 Read 5 Done Sent Done false
42742024-02-07T18:56:13.321Z 28 Acked 1027 Read 1 Done Sent Done false
42752024-02-07T18:56:13.321Z 29 Acked 1028 Write 9 Done Sent Done false
42762024-02-07T18:56:13.321Z 30 Acked 1029 Read 3 Done Sent Sent false
42772024-02-07T18:56:13.321Z STATES DS:0 DS:1 DS:2 TOTAL
42782024-02-07T18:56:13.321Z New 0 0 0 0
42792024-02-07T18:56:13.321Z Sent 0 30 1 31
42802024-02-07T18:56:13.321Z Done 30 0 29 59
42812024-02-07T18:56:13.321Z Skipped 0 0 0 0
42822024-02-07T18:56:13.321Z Error 0 0 0 0
42832024-02-07T18:56:13.321ZLast Flush: 1019 0 1019
42842024-02-07T18:56:13.321ZDownstairs last five completed:
42852024-02-07T18:56:13.321ZUpstairs last five completed: 30 29 28 27 26
42862024-02-07T18:56:13.321Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:12.578651461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1287}
42872024-02-07T18:56:13.321ZWrote out file "/tmp/test_repair_verify.data"
42882024-02-07T18:56:13.324Z
42892024-02-07T18:56:13.327ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42902024-02-07T18:56:13.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42912024-02-07T18:56:13.341Z 0 000-019 36 36 36 67 67 67 F F F
42922024-02-07T18:56:13.341Z 1 020-039 36 36 36 69 69 69 F F F
42932024-02-07T18:56:13.341Z 2 040-059 30 30 30 56 56 56 F F F
42942024-02-07T18:56:13.341Z 3 060-079 38 30 38 71 56 71 F F F <---
42952024-02-07T18:56:13.341Z 4 080-099 32 32 32 59 59 59 F F F
42962024-02-07T18:56:13.341Z 5 100-119 37 37 37 70 70 70 F F F
42972024-02-07T18:56:13.342Z 6 120-139 34 34 34 65 65 65 F F F
42982024-02-07T18:56:13.342Z 7 140-159 37 37 37 70 70 70 F F F
42992024-02-07T18:56:13.342Z 8 160-179 27 27 27 51 51 51 F F F
43002024-02-07T18:56:13.345Z 9 180-199 33 33 33 61 61 61 F F F
43012024-02-07T18:56:13.345Z 10 200-219 38 36 38 71 68 71 F F F <---
43022024-02-07T18:56:13.345Z 11 220-239 37 37 37 70 70 70 F F F
43032024-02-07T18:56:13.345Z 12 240-259 38 37 38 72 70 72 F F F <---
43042024-02-07T18:56:13.345Z 13 260-279 38 37 38 72 70 72 F F F <---
43052024-02-07T18:56:13.345Z 14 280-299 34 34 34 62 62 62 F F F
43062024-02-07T18:56:13.345Z 15 300-319 33 33 33 61 61 61 T F T <---
43072024-02-07T18:56:13.345Z 16 320-339 24 24 24 45 45 45 T F T <---
43082024-02-07T18:56:13.345Z 17 340-359 34 34 34 64 64 64 F F F
43092024-02-07T18:56:13.345Z 18 360-379 38 33 38 71 61 71 F F F <---
43102024-02-07T18:56:13.345Z 19 380-399 38 36 38 71 69 71 F F F <---
43112024-02-07T18:56:13.345Z 20 400-419 34 34 34 64 64 64 F F F
43122024-02-07T18:56:13.345Z 21 420-439 36 36 36 67 67 67 F F F
43132024-02-07T18:56:13.346Z 22 440-459 37 37 37 70 70 70 F F F
43142024-02-07T18:56:13.346Z 23 460-479 34 34 34 65 65 65 F F F
43152024-02-07T18:56:13.346Z 24 480-499 31 31 31 57 57 57 F F F
43162024-02-07T18:56:13.346Z 25 500-519 32 32 32 59 59 59 F F F
43172024-02-07T18:56:13.346Z 26 520-539 32 32 32 60 60 60 T F T <---
43182024-02-07T18:56:13.346Z 27 540-559 36 36 36 67 67 67 T F T <---
43192024-02-07T18:56:13.346Z 28 560-579 32 32 32 60 60 60 F F F
43202024-02-07T18:56:13.346Z 29 580-599 34 34 34 62 62 62 F F F
43212024-02-07T18:56:13.346ZMax gen: 38, Max flush: 72
43222024-02-07T18:56:13.346ZError: Difference in extent metadata found!
43232024-02-07T18:56:13.346ZA Difference in extent metadata is expected here
43242024-02-07T18:56:13.346ZOn loop 19
43252024-02-07T18:56:13.346Z
43262024-02-07T18:56:13.349ZVerifying data now
43272024-02-07T18:56:17.854ZNo change after verify
43282024-02-07T18:56:17.854ZLoop: 19 Downstairs dump after verify (and repair):
43292024-02-07T18:56:17.869ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43302024-02-07T18:56:17.869Z 0 000-019 36 36 36 67 67 67 F F F
43312024-02-07T18:56:17.869Z 1 020-039 36 36 36 69 69 69 F F F
43322024-02-07T18:56:17.870Z 2 040-059 30 30 30 56 56 56 F F F
43332024-02-07T18:56:17.870Z 3 060-079 38 38 38 71 71 71 F F F
43342024-02-07T18:56:17.870Z 4 080-099 32 32 32 59 59 59 F F F
43352024-02-07T18:56:17.870Z 5 100-119 37 37 37 70 70 70 F F F
43362024-02-07T18:56:17.870Z 6 120-139 34 34 34 65 65 65 F F F
43372024-02-07T18:56:17.870Z 7 140-159 37 37 37 70 70 70 F F F
43382024-02-07T18:56:17.870Z 8 160-179 27 27 27 51 51 51 F F F
43392024-02-07T18:56:17.870Z 9 180-199 33 33 33 61 61 61 F F F
43402024-02-07T18:56:17.870Z 10 200-219 38 38 38 71 71 71 F F F
43412024-02-07T18:56:17.870Z 11 220-239 37 37 37 70 70 70 F F F
43422024-02-07T18:56:17.870Z 12 240-259 38 38 38 72 72 72 F F F
43432024-02-07T18:56:17.870Z 13 260-279 38 38 38 72 72 72 F F F
43442024-02-07T18:56:17.873Z 14 280-299 34 34 34 62 62 62 F F F
43452024-02-07T18:56:17.873Z 15 300-319 39 39 39 73 73 73 F F F
43462024-02-07T18:56:17.873Z 16 320-339 39 39 39 73 73 73 F F F
43472024-02-07T18:56:17.873Z 17 340-359 34 34 34 64 64 64 F F F
43482024-02-07T18:56:17.874Z 18 360-379 38 38 38 71 71 71 F F F
43492024-02-07T18:56:17.874Z 19 380-399 38 38 38 71 71 71 F F F
43502024-02-07T18:56:17.874Z 20 400-419 34 34 34 64 64 64 F F F
43512024-02-07T18:56:17.874Z 21 420-439 36 36 36 67 67 67 F F F
43522024-02-07T18:56:17.874Z 22 440-459 37 37 37 70 70 70 F F F
43532024-02-07T18:56:17.874Z 23 460-479 34 34 34 65 65 65 F F F
43542024-02-07T18:56:17.874Z 24 480-499 31 31 31 57 57 57 F F F
43552024-02-07T18:56:17.874Z 25 500-519 32 32 32 59 59 59 F F F
43562024-02-07T18:56:17.874Z 26 520-539 39 39 39 73 73 73 F F F
43572024-02-07T18:56:17.874Z 27 540-559 39 39 39 73 73 73 F F F
43582024-02-07T18:56:17.874Z 28 560-579 32 32 32 60 60 60 F F F
43592024-02-07T18:56:17.874Z 29 580-599 34 34 34 62 62 62 F F F
43602024-02-07T18:56:17.874ZMax gen: 39, Max flush: 73
43612024-02-07T18:56:17.874Z
43622024-02-07T18:56:17.874ZBegin loop 20
43632024-02-07T18:56:17.874ZDownstairs to restart: 0
43642024-02-07T18:56:17.891Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15165686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295}
43652024-02-07T18:56:17.898Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15195972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295}
43662024-02-07T18:56:17.898ZCrucible runtime is spawned
43672024-02-07T18:56:17.898Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152004613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295}
43682024-02-07T18:56:17.898Z{"msg":"Crucible 0f79868a-9c97-41a4-ab7a-ea6e823713bc has session id: c9f51dd8-8710-42f7-9667-19f4b7772942","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152040609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43692024-02-07T18:56:17.898Z{"msg":"Upstairs opts: Upstairs UUID: 0f79868a-9c97-41a4-ab7a-ea6e823713bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152098341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43702024-02-07T18:56:17.898Z{"msg":"Crucible stats registered with UUID: 0f79868a-9c97-41a4-ab7a-ea6e823713bc","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152133646Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43712024-02-07T18:56:17.898Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152183449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295}
43722024-02-07T18:56:17.898Z{"msg":"0f79868a-9c97-41a4-ab7a-ea6e823713bc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152228183Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43732024-02-07T18:56:17.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152274657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43742024-02-07T18:56:17.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152320521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43752024-02-07T18:56:17.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152355826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43762024-02-07T18:56:17.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152451313Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43772024-02-07T18:56:17.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152498267Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43782024-02-07T18:56:17.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15254744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43792024-02-07T18:56:17.898Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152588685Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43802024-02-07T18:56:17.898Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152609572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43812024-02-07T18:56:17.898Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152641827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43822024-02-07T18:56:17.898Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152658915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43832024-02-07T18:56:17.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152691361Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43842024-02-07T18:56:17.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.152708718Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43852024-02-07T18:56:17.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.153187544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43862024-02-07T18:56:17.899Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15321623Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43872024-02-07T18:56:17.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.153233187Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43882024-02-07T18:56:17.899Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.153272742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43892024-02-07T18:56:17.899Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.153819138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43902024-02-07T18:56:17.899Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154104329Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43912024-02-07T18:56:17.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154152653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43922024-02-07T18:56:17.899Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154179739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43932024-02-07T18:56:17.899Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154364065Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43942024-02-07T18:56:17.899Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154431705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43952024-02-07T18:56:17.899Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154462161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43962024-02-07T18:56:17.899Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15454392Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43972024-02-07T18:56:17.899Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154589514Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43982024-02-07T18:56:17.899Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154622739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
43992024-02-07T18:56:17.904Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.154846899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44002024-02-07T18:56:17.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155349421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44012024-02-07T18:56:17.904Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155389586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44022024-02-07T18:56:17.904Z{"msg":"[0]R flush_numbers[0..12]: [67, 69, 56, 71, 59, 70, 65, 70, 51, 61, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155407263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44032024-02-07T18:56:17.904Z{"msg":"[0]R generation[0..12]: [36, 36, 30, 38, 32, 37, 34, 37, 27, 33, 38, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15542965Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44042024-02-07T18:56:17.904Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155452327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44052024-02-07T18:56:17.904Z{"msg":"[1]R flush_numbers[0..12]: [67, 69, 56, 71, 59, 70, 65, 70, 51, 61, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155467405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44062024-02-07T18:56:17.904Z{"msg":"[1]R generation[0..12]: [36, 36, 30, 38, 32, 37, 34, 37, 27, 33, 38, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155488182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44072024-02-07T18:56:17.904Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15550305Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44082024-02-07T18:56:17.904Z{"msg":"[2]R flush_numbers[0..12]: [67, 69, 56, 71, 59, 70, 65, 70, 51, 61, 71, 70]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
44092024-02-07T18:56:17.904Z2024-02-07T18:56:17.155525028Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44102024-02-07T18:56:17.904Z{"msg":"[2]R generation[0..12]: [36, 36, 30, 38, 32, 37, 34, 37, 27, 33, 38, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155543365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44112024-02-07T18:56:17.904Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155575211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44122024-02-07T18:56:17.904Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155605586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44132024-02-07T18:56:17.904Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155620375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44142024-02-07T18:56:17.905Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155641872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44152024-02-07T18:56:17.905Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15565561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44162024-02-07T18:56:17.905Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155676037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44172024-02-07T18:56:17.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155689175Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44182024-02-07T18:56:17.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155702583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44192024-02-07T18:56:17.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155723801Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44202024-02-07T18:56:17.905Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155738288Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44212024-02-07T18:56:17.905Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155757646Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44222024-02-07T18:56:17.905Z{"msg":"0f79868a-9c97-41a4-ab7a-ea6e823713bc is now active with session: c9f51dd8-8710-42f7-9667-19f4b7772942","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155770994Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44232024-02-07T18:56:17.905Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.155789862Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295,"session_id":"c9f51dd8-8710-42f7-9667-19f4b7772942"}
44242024-02-07T18:56:17.905Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:17.15580272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295}
44252024-02-07T18:56:17.905ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
44262024-02-07T18:56:17.905ZLoading write count information from file "/tmp/test_repair_verify.data"
44272024-02-07T18:56:17.905ZRun Repair workload
44282024-02-07T18:56:17.905Z01/30 Write block 391 len 4096 data: 4
44292024-02-07T18:56:17.905Z02/30 Write block 104 len 32768 data: 5 5 5 5 5 7 5 5
44302024-02-07T18:56:17.905Z03/30 Read block 114 len 40960
44312024-02-07T18:56:17.905Z04/30 Read block 562 len 24576
44322024-02-07T18:56:17.905Z05/30 Write block 569 len 12288 data: 2 2 2
44332024-02-07T18:56:17.905Z06/30 Write block 1 len 36864 data: 2 2 2 2 3 2 3 3 4
44342024-02-07T18:56:17.905Z07/30 Write block 235 len 16384 data: 9 7 6 6
44352024-02-07T18:56:17.912Z08/30 Flush
44362024-02-07T18:56:17.912Z09/30 Write block 211 len 20480 data: 3 3 4 4 3
44372024-02-07T18:56:17.912Z10/30 Write block 302 len 16384 data: 2 2 4 4
44382024-02-07T18:56:17.912Z11/30 Write block 174 len 4096 data: 5
44392024-02-07T18:56:17.912Z12/30 Read block 200 len 20480
44402024-02-07T18:56:17.912Z13/30 Read block 478 len 40960
44412024-02-07T18:56:17.915Z14/30 Read block 571 len 4096
44422024-02-07T18:56:17.916Z15/30 Write block 231 len 28672 data: 4 5 6 7 10 8 7
44432024-02-07T18:56:17.916Z16/30 Read block 173 len 32768
44442024-02-07T18:56:17.916Z17/30 Write block 360 len 32768 data: 5 6 7 6 5 4 5 4
44452024-02-07T18:56:17.916Z18/30 Write block 535 len 28672 data: 4 4 4 4 4 4 5
44462024-02-07T18:56:17.916Z19/30 Read block 102 len 36864
44472024-02-07T18:56:17.920Z20/30 Write block 297 len 8192 data: 4 3
44482024-02-07T18:56:17.920Z21/30 Read block 91 len 12288
44492024-02-07T18:56:17.920Z22/30 Write block 370 len 20480 data: 3 3 2 2 2
44502024-02-07T18:56:17.920Z23/30 Flush
44512024-02-07T18:56:17.924Z24/30 Write block 262 len 24576 data: 7 6 6 5 3 2
44522024-02-07T18:56:17.924Z25/30 Read block 394 len 40960
44532024-02-07T18:56:17.924Z26/30 Write block 238 len 16384 data: 7 6 6 6
44542024-02-07T18:56:17.924Z27/30 Write block 141 len 28672 data: 4 4 4 4 4 6 6
44552024-02-07T18:56:17.927Z28/30 Write block 351 len 24576 data: 5 3 2 4 4 2
44562024-02-07T18:56:17.927Z29/30 Write block 333 len 16384 data: 4 4 5 5
44572024-02-07T18:56:17.927Z30/30 Read block 103 len 20480
44582024-02-07T18:56:17.927Z----------------------------------------------------------------
44592024-02-07T18:56:17.927Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:22
44602024-02-07T18:56:17.927ZStates: Active Active Active
44612024-02-07T18:56:17.927ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44622024-02-07T18:56:17.927Z 9 Acked 1008 Write 5 Done Done Done false
44632024-02-07T18:56:17.928Z 10 Acked 1009 Write 4 Done Done Done false
44642024-02-07T18:56:17.928Z 11 Acked 1010 Write 1 Done Done Done false
44652024-02-07T18:56:17.928Z 12 Acked 1011 Read 5 Done Done Done false
44662024-02-07T18:56:17.928Z 13 Acked 1012 Read 10 Done Done Done false
44672024-02-07T18:56:17.928Z 14 Acked 1013 Read 1 Sent Done Done false
44682024-02-07T18:56:17.928Z 15 Acked 1014 Write 7 Sent Done Done false
44692024-02-07T18:56:17.928Z 16 Acked 1015 Read 8 Sent Done Done false
44702024-02-07T18:56:17.928Z 17 Acked 1016 Write 8 Sent Done Done false
44712024-02-07T18:56:17.928Z 18 Acked 1017 Write 7 Sent Done Done false
44722024-02-07T18:56:17.928Z 19 Acked 1018 Read 9 Sent Done Done false
44732024-02-07T18:56:17.928Z 20 Acked 1019 Write 2 Sent Done Done false
44742024-02-07T18:56:17.928Z 21 Acked 1020 Read 3 Sent Done Done false
44752024-02-07T18:56:17.928Z 22 Acked 1021 Write 5 Sent Done Done false
44762024-02-07T18:56:17.928Z 23 Acked 1022 Flush 0 Sent Done Done false
44772024-02-07T18:56:17.928Z 24 Acked 1023 Write 6 Sent Done Done false
44782024-02-07T18:56:17.928Z 25 Acked 1024 Read 10 Sent Done Done false
44792024-02-07T18:56:17.928Z 26 Acked 1025 Write 4 Sent Done Done false
44802024-02-07T18:56:17.928Z 27 Acked 1026 Write 7 Sent Done Done false
44812024-02-07T18:56:17.928Z 28 Acked 1027 Write 6 Sent Done Done false
44822024-02-07T18:56:17.928Z 29 Acked 1028 Write 4 Sent Done Done false
44832024-02-07T18:56:17.928Z 30 Acked 1029 Read 5 Sent Sent Done false
44842024-02-07T18:56:17.928Z STATES DS:0 DS:1 DS:2 TOTAL
44852024-02-07T18:56:17.928Z New 0 0 0 0
44862024-02-07T18:56:17.928Z Sent 17 1 0 18
44872024-02-07T18:56:17.928Z Done 5 21 22 48
44882024-02-07T18:56:17.929Z Skipped 0 0 0 0
44892024-02-07T18:56:17.929Z Error 0 0 0 0
44902024-02-07T18:56:17.929ZLast Flush: 1007 1022 1022
44912024-02-07T18:56:17.929ZDownstairs last five completed: 1007 1006 1005 1004 1003
44922024-02-07T18:56:17.929ZUpstairs last five completed: 30 29 28 27 26
44932024-02-07T18:56:17.929Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:17.186747845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1295}
44942024-02-07T18:56:17.929ZWrote out file "/tmp/test_repair_verify.data"
44952024-02-07T18:56:17.932Z
44962024-02-07T18:56:17.935ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44972024-02-07T18:56:17.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44982024-02-07T18:56:17.952Z 0 000-019 40 40 40 74 74 74 F F F
44992024-02-07T18:56:17.952Z 1 020-039 36 36 36 69 69 69 F F F
45002024-02-07T18:56:17.952Z 2 040-059 30 30 30 56 56 56 F F F
45012024-02-07T18:56:17.952Z 3 060-079 38 38 38 71 71 71 F F F
45022024-02-07T18:56:17.952Z 4 080-099 32 32 32 59 59 59 F F F
45032024-02-07T18:56:17.952Z 5 100-119 40 40 40 74 74 74 F F F
45042024-02-07T18:56:17.952Z 6 120-139 34 34 34 65 65 65 F F F
45052024-02-07T18:56:17.952Z 7 140-159 37 37 37 70 70 70 F T T <---
45062024-02-07T18:56:17.955Z 8 160-179 27 40 40 51 75 75 T F F <---
45072024-02-07T18:56:17.955Z 9 180-199 33 33 33 61 61 61 F F F
45082024-02-07T18:56:17.955Z 10 200-219 38 40 40 71 75 75 T F F <---
45092024-02-07T18:56:17.955Z 11 220-239 40 40 40 74 75 75 F T T <---
45102024-02-07T18:56:17.955Z 12 240-259 38 38 38 72 72 72 F T T <---
45112024-02-07T18:56:17.955Z 13 260-279 38 38 38 72 72 72 F T T <---
45122024-02-07T18:56:17.955Z 14 280-299 34 40 40 62 75 75 F F F <---
45132024-02-07T18:56:17.955Z 15 300-319 39 40 40 73 75 75 T F F <---
45142024-02-07T18:56:17.956Z 16 320-339 39 39 39 73 73 73 F T T <---
45152024-02-07T18:56:17.956Z 17 340-359 34 34 34 64 64 64 F T T <---
45162024-02-07T18:56:17.956Z 18 360-379 38 40 40 71 75 75 F F F <---
45172024-02-07T18:56:17.956Z 19 380-399 40 40 40 74 74 74 F F F
45182024-02-07T18:56:17.956Z 20 400-419 34 34 34 64 64 64 F F F
45192024-02-07T18:56:17.956Z 21 420-439 36 36 36 67 67 67 F F F
45202024-02-07T18:56:17.956Z 22 440-459 37 37 37 70 70 70 F F F
45212024-02-07T18:56:17.956Z 23 460-479 34 34 34 65 65 65 F F F
45222024-02-07T18:56:17.956Z 24 480-499 31 31 31 57 57 57 F F F
45232024-02-07T18:56:17.956Z 25 500-519 32 32 32 59 59 59 F F F
45242024-02-07T18:56:17.956Z 26 520-539 39 40 40 73 75 75 F F F <---
45252024-02-07T18:56:17.956Z 27 540-559 39 40 40 73 75 75 F F F <---
45262024-02-07T18:56:17.956Z 28 560-579 40 40 40 74 74 74 F F F
45272024-02-07T18:56:17.956Z 29 580-599 34 34 34 62 62 62 F F F
45282024-02-07T18:56:17.956ZMax gen: 40, Max flush: 75
45292024-02-07T18:56:17.956ZError: Difference in extent metadata found!
45302024-02-07T18:56:17.956ZA Difference in extent metadata is expected here
45312024-02-07T18:56:17.956ZOn loop 20
45322024-02-07T18:56:17.956Z
45332024-02-07T18:56:17.959ZVerifying data now
45342024-02-07T18:56:23.598ZNo change after verify
45352024-02-07T18:56:23.598ZLoop: 20 Downstairs dump after verify (and repair):
45362024-02-07T18:56:23.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45372024-02-07T18:56:23.617Z 0 000-019 40 40 40 74 74 74 F F F
45382024-02-07T18:56:23.617Z 1 020-039 36 36 36 69 69 69 F F F
45392024-02-07T18:56:23.617Z 2 040-059 30 30 30 56 56 56 F F F
45402024-02-07T18:56:23.617Z 3 060-079 38 38 38 71 71 71 F F F
45412024-02-07T18:56:23.617Z 4 080-099 32 32 32 59 59 59 F F F
45422024-02-07T18:56:23.617Z 5 100-119 40 40 40 74 74 74 F F F
45432024-02-07T18:56:23.617Z 6 120-139 34 34 34 65 65 65 F F F
45442024-02-07T18:56:23.626Z 7 140-159 41 41 41 76 76 76 F F F
45452024-02-07T18:56:23.626Z 8 160-179 40 40 40 75 75 75 F F F
45462024-02-07T18:56:23.626Z 9 180-199 33 33 33 61 61 61 F F F
45472024-02-07T18:56:23.626Z 10 200-219 40 40 40 75 75 75 F F F
45482024-02-07T18:56:23.626Z 11 220-239 41 41 41 76 76 76 F F F
45492024-02-07T18:56:23.626Z 12 240-259 41 41 41 76 76 76 F F F
45502024-02-07T18:56:23.626Z 13 260-279 41 41 41 76 76 76 F F F
45512024-02-07T18:56:23.626Z 14 280-299 40 40 40 75 75 75 F F F
45522024-02-07T18:56:23.626Z 15 300-319 40 40 40 75 75 75 F F F
45532024-02-07T18:56:23.626Z 16 320-339 41 41 41 76 76 76 F F F
45542024-02-07T18:56:23.626Z 17 340-359 41 41 41 76 76 76 F F F
45552024-02-07T18:56:23.626Z 18 360-379 40 40 40 75 75 75 F F F
45562024-02-07T18:56:23.626Z 19 380-399 40 40 40 74 74 74 F F F
45572024-02-07T18:56:23.626Z 20 400-419 34 34 34 64 64 64 F F F
45582024-02-07T18:56:23.626Z 21 420-439 36 36 36 67 67 67 F F F
45592024-02-07T18:56:23.626Z 22 440-459 37 37 37 70 70 70 F F F
45602024-02-07T18:56:23.626Z 23 460-479 34 34 34 65 65 65 F F F
45612024-02-07T18:56:23.627Z 24 480-499 31 31 31 57 57 57 F F F
45622024-02-07T18:56:23.627Z 25 500-519 32 32 32 59 59 59 F F F
45632024-02-07T18:56:23.627Z 26 520-539 40 40 40 75 75 75 F F F
45642024-02-07T18:56:23.627Z 27 540-559 40 40 40 75 75 75 F F F
45652024-02-07T18:56:23.627Z 28 560-579 40 40 40 74 74 74 F F F
45662024-02-07T18:56:23.627Z 29 580-599 34 34 34 62 62 62 F F F
45672024-02-07T18:56:23.627ZMax gen: 41, Max flush: 76
45682024-02-07T18:56:23.627Z
45692024-02-07T18:56:23.627ZBegin loop 21
45702024-02-07T18:56:23.627ZDownstairs to restart: 0
45712024-02-07T18:56:23.633Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892246542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303}
45722024-02-07T18:56:23.633Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892575477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303}
45732024-02-07T18:56:23.633Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
45742024-02-07T18:56:23.633Z,"time":"2024-02-07T18:56:22.892610872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303}
45752024-02-07T18:56:23.647Z{"msg":"Crucible cd8f6c80-a699-4c06-895a-ecdf359acfa8 has session id: 65d289a4-5dcd-44dd-8118-3583918996e0","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892639119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45762024-02-07T18:56:23.647Z{"msg":"Upstairs opts: Upstairs UUID: cd8f6c80-a699-4c06-895a-ecdf359acfa8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892675464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45772024-02-07T18:56:23.647Z{"msg":"Crucible stats registered with UUID: cd8f6c80-a699-4c06-895a-ecdf359acfa8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.8927018Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45782024-02-07T18:56:23.647Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892733506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303}
45792024-02-07T18:56:23.647Z{"msg":"cd8f6c80-a699-4c06-895a-ecdf359acfa8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892763912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45802024-02-07T18:56:23.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892793478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45812024-02-07T18:56:23.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892815215Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45822024-02-07T18:56:23.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892835752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45832024-02-07T18:56:23.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892858459Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45842024-02-07T18:56:23.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892881886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45852024-02-07T18:56:23.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892906812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45862024-02-07T18:56:23.648Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892944498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45872024-02-07T18:56:23.648Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.892973734Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45882024-02-07T18:56:23.648Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.893012738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45892024-02-07T18:56:23.648Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.893191274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45902024-02-07T18:56:23.648Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.893250006Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45912024-02-07T18:56:23.648Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.893276273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45922024-02-07T18:56:23.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.894001724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45932024-02-07T18:56:23.648Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.89403897Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45942024-02-07T18:56:23.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.894054647Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45952024-02-07T18:56:23.648Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.894068685Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45962024-02-07T18:56:23.648Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.894693761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45972024-02-07T18:56:23.648Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.894745684Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45982024-02-07T18:56:23.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895301669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
45992024-02-07T18:56:23.648Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895340593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46002024-02-07T18:56:23.648Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895357471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46012024-02-07T18:56:23.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895377998Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46022024-02-07T18:56:23.648Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895392156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46032024-02-07T18:56:23.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895412964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46042024-02-07T18:56:23.648Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895427412Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46052024-02-07T18:56:23.648Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895447179Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46062024-02-07T18:56:23.648Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.895967299Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46072024-02-07T18:56:23.660Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896528453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46082024-02-07T18:56:23.660Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896568107Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46092024-02-07T18:56:23.661Z{"msg":"[0]R flush_numbers[0..12]: [74, 69, 56, 71, 59, 74, 65, 76, 75, 61, 75, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896615301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46102024-02-07T18:56:23.661Z{"msg":"[0]R generation[0..12]: [40, 36, 30, 38, 32, 40, 34, 41, 40, 33, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896638218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46112024-02-07T18:56:23.661Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896654326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46122024-02-07T18:56:23.661ZWait for a query_work_queue command to finish before sending IO
46132024-02-07T18:56:23.661Z{"msg":"[1]R flush_numbers[0..12]: [74, 69, 56, 71, 59, 74, 65, 76, 75, 61, 75, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896680882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46142024-02-07T18:56:23.661Z{"msg":"[1]R generation[0..12]: [40, 36, 30, 38, 32, 40, 34, 41, 40, 33, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.89669517Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46152024-02-07T18:56:23.661Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896722926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46162024-02-07T18:56:23.661Z{"msg":"[2]R flush_numbers[0..12]: [74, 69, 56, 71, 59, 74, 65, 76, 75, 61, 75, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896738634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46172024-02-07T18:56:23.661Z{"msg":"[2]R generation[0..12]: [40, 36, 30, 38, 32, 40, 34, 41, 40, 33, 40, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896754322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46182024-02-07T18:56:23.661Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.89676867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46192024-02-07T18:56:23.661Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896782618Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46202024-02-07T18:56:23.661Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896803865Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46212024-02-07T18:56:23.661Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896817504Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46222024-02-07T18:56:23.661Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896830802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46232024-02-07T18:56:23.661Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896851899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46242024-02-07T18:56:23.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896865627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46252024-02-07T18:56:23.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896879685Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46262024-02-07T18:56:23.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896900442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46272024-02-07T18:56:23.661Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896914321Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46282024-02-07T18:56:23.661Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896934058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46292024-02-07T18:56:23.661Z{"msg":"cd8f6c80-a699-4c06-895a-ecdf359acfa8 is now active with session: 65d289a4-5dcd-44dd-8118-3583918996e0","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896948696Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46302024-02-07T18:56:23.661Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896970273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303,"session_id":"65d289a4-5dcd-44dd-8118-3583918996e0"}
46312024-02-07T18:56:23.662Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:22.896998559Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303}
46322024-02-07T18:56:23.662ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46332024-02-07T18:56:23.662ZLoading write count information from file "/tmp/test_repair_verify.data"
46342024-02-07T18:56:23.662ZRun Repair workload
46352024-02-07T18:56:23.662Z01/30 Write block 527 len 8192 data: 4 3
46362024-02-07T18:56:23.662Z02/30 Read block 134 len 24576
46372024-02-07T18:56:23.662Z03/30 Read block 40 len 8192
46382024-02-07T18:56:23.662Z04/30 Write block 228 len 8192 data: 3 3
46392024-02-07T18:56:23.662Z05/30 Read block 355 len 12288
46402024-02-07T18:56:23.662Z06/30 Write block 308 len 16384 data: 4 4 4 4
46412024-02-07T18:56:23.662Z07/30 Read block 199 len 28672
46422024-02-07T18:56:23.662Z08/30 Read block 446 len 8192
46432024-02-07T18:56:23.662Z09/30 Read block 488 len 8192
46442024-02-07T18:56:23.662Z10/30 Write block 537 len 16384 data: 5 5 5 5
46452024-02-07T18:56:23.662Z11/30 Write block 550 len 8192 data: 3 3
46462024-02-07T18:56:23.662Z12/30 Write block 510 len 4096 data: 5
46472024-02-07T18:56:23.662Z13/30 Write block 19 len 20480 data: 7 7 6 7 8
46482024-02-07T18:56:23.662Z14/30 Write block 476 len 4096 data: 3
46492024-02-07T18:56:23.662Z15/30 Flush
46502024-02-07T18:56:23.662Z16/30 Write block 132 len 16384 data: 2 2 2 2
46512024-02-07T18:56:23.662Z17/30 Write block 19 len 28672 data: 8 8 7 8 9 8 7
46522024-02-07T18:56:23.662Z18/30 Read block 126 len 28672
46532024-02-07T18:56:23.662Z19/30 Read block 275 len 36864
46542024-02-07T18:56:23.668Z20/30 Flush
46552024-02-07T18:56:23.668Z21/30 Write block 125 len 40960 data: 3 3 3 4 4 3 3 3 3 3
46562024-02-07T18:56:23.668Z22/30 Read block 334 len 36864
46572024-02-07T18:56:23.668Z23/30 Read block 562 len 32768
46582024-02-07T18:56:23.668Z24/30 Write block 316 len 4096 data: 6
46592024-02-07T18:56:23.669Z25/30 Write block 502 len 24576 data: 4 5 5 5 4 4
46602024-02-07T18:56:23.669Z26/30 Read block 448 len 16384
46612024-02-07T18:56:23.669Z27/30 Read block 547 len 28672
46622024-02-07T18:56:23.669Z28/30 Write block 12 len 24576 data: 3 4 5 4 5 6
46632024-02-07T18:56:23.669Z29/30 Read block 549 len 4096
46642024-02-07T18:56:23.669Z30/30 Write block 149 len 24576 data: 7 6 6 6 6 5
46652024-02-07T18:56:23.669Z----------------------------------------------------------------
46662024-02-07T18:56:23.669Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46672024-02-07T18:56:23.669ZStates: Active Active Active
46682024-02-07T18:56:23.669ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46692024-02-07T18:56:23.669Z 1 Acked 1000 Write 2 Done Done Done false
46702024-02-07T18:56:23.669Z 2 Acked 1001 Read 6 Done Done Done false
46712024-02-07T18:56:23.669Z 3 Acked 1002 Read 2 Done Done Done false
46722024-02-07T18:56:23.669Z 4 Acked 1003 Write 2 Sent Done Done false
46732024-02-07T18:56:23.669Z 5 Acked 1004 Read 3 Sent Done Done false
46742024-02-07T18:56:23.669Z 6 Acked 1005 Write 4 Sent Done Done false
46752024-02-07T18:56:23.669Z 7 Acked 1006 Read 7 Sent Done Done false
46762024-02-07T18:56:23.670Z 8 Acked 1007 Read 2 Sent Done Done false
46772024-02-07T18:56:23.670Z 9 Acked 1008 Read 2 Sent Done Done false
46782024-02-07T18:56:23.670Z 10 Acked 1009 Write 4 Sent Done Done false
46792024-02-07T18:56:23.670Z 11 Acked 1010 Write 2 Sent Done Done false
46802024-02-07T18:56:23.670Z 12 Acked 1011 Write 1 Sent Done Done false
46812024-02-07T18:56:23.670Z 13 Acked 1012 Write 5 Sent Done Done false
46822024-02-07T18:56:23.670Z 14 Acked 1013 Write 1 Sent Done Done false
46832024-02-07T18:56:23.670Z 15 Acked 1014 Flush 0 Sent Done Done false
46842024-02-07T18:56:23.670Z 16 Acked 1015 Write 4 Sent Done Done false
46852024-02-07T18:56:23.671Z 17 Acked 1016 Write 7 Sent Done Done false
46862024-02-07T18:56:23.671Z 18 Acked 1017 Read 7 Sent Done Done false
46872024-02-07T18:56:23.671Z 19 Acked 1018 Read 9 Sent Done Done false
46882024-02-07T18:56:23.671Z 20 Acked 1019 Flush 0 Sent Done Done false
46892024-02-07T18:56:23.671Z 21 Acked 1020 Write 10 Sent Done Done false
46902024-02-07T18:56:23.671Z 22 Acked 1021 Read 9 Sent Done Done false
46912024-02-07T18:56:23.672Z 23 Acked 1022 Read 8 Sent Done Done false
46922024-02-07T18:56:23.672Z 24 Acked 1023 Write 1 Sent Done Done false
46932024-02-07T18:56:23.672Z 25 Acked 1024 Write 6 Sent Done Done false
46942024-02-07T18:56:23.672Z 26 Acked 1025 Read 4 Sent Done Done false
46952024-02-07T18:56:23.672Z 27 Acked 1026 Read 7 Sent Done Done false
46962024-02-07T18:56:23.672Z 28 Acked 1027 Write 6 Sent Done Done false
46972024-02-07T18:56:23.672Z 29 Acked 1028 Read 1 Sent Done Done false
46982024-02-07T18:56:23.672Z 30 Acked 1029 Write 6 Sent Sent Sent false
46992024-02-07T18:56:23.672Z STATES DS:0 DS:1 DS:2 TOTAL
47002024-02-07T18:56:23.673Z New 0 0 0 0
47012024-02-07T18:56:23.673Z Sent 27 1 1 29
47022024-02-07T18:56:23.673Z Done 3 29 29 61
47032024-02-07T18:56:23.673Z Skipped 0 0 0 0
47042024-02-07T18:56:23.673Z Error 0 0 0 0
47052024-02-07T18:56:23.673ZLast Flush: 0 1019 1019
47062024-02-07T18:56:23.673ZDownstairs last five completed:
47072024-02-07T18:56:23.674ZUpstairs last five completed: 30 29 28 27 26
47082024-02-07T18:56:23.674Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:22.925509923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1303}
47092024-02-07T18:56:23.674ZWrote out file "/tmp/test_repair_verify.data"
47102024-02-07T18:56:24.087Z
47112024-02-07T18:56:24.087ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
47122024-02-07T18:56:24.098ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47132024-02-07T18:56:24.098Z 0 000-019 40 42 42 74 78 78 F T T <---
47142024-02-07T18:56:24.099Z 1 020-039 36 42 42 69 78 78 F F F <---
47152024-02-07T18:56:24.099Z 2 040-059 30 30 30 56 56 56 F F F
47162024-02-07T18:56:24.099Z 3 060-079 38 38 38 71 71 71 F F F
47172024-02-07T18:56:24.099Z 4 080-099 32 32 32 59 59 59 F F F
47182024-02-07T18:56:24.099Z 5 100-119 40 40 40 74 74 74 F F F
47192024-02-07T18:56:24.099Z 6 120-139 34 42 42 65 78 78 F T T <---
47202024-02-07T18:56:24.099Z 7 140-159 41 41 41 76 76 76 F T T <---
47212024-02-07T18:56:24.099Z 8 160-179 40 40 40 75 75 75 F F F
47222024-02-07T18:56:24.099Z 9 180-199 33 33 33 61 61 61 F F F
47232024-02-07T18:56:24.099Z 10 200-219 40 40 40 75 75 75 F F F
47242024-02-07T18:56:24.099Z 11 220-239 41 42 42 76 77 77 F F F <---
47252024-02-07T18:56:24.099Z 12 240-259 41 41 41 76 76 76 F F F
47262024-02-07T18:56:24.099Z 13 260-279 41 41 41 76 76 76 F F F
47272024-02-07T18:56:24.099Z 14 280-299 40 40 40 75 75 75 F F F
47282024-02-07T18:56:24.099Z 15 300-319 40 42 42 75 77 77 F T T <---
47292024-02-07T18:56:24.099Z 16 320-339 41 41 41 76 76 76 F F F
47302024-02-07T18:56:24.099Z 17 340-359 41 41 41 76 76 76 F F F
47312024-02-07T18:56:24.099Z 18 360-379 40 40 40 75 75 75 F F F
47322024-02-07T18:56:24.099Z 19 380-399 40 40 40 74 74 74 F F F
47332024-02-07T18:56:24.099Z 20 400-419 34 34 34 64 64 64 F F F
47342024-02-07T18:56:24.100Z 21 420-439 36 36 36 67 67 67 F F F
47352024-02-07T18:56:24.100Z 22 440-459 37 37 37 70 70 70 F F F
47362024-02-07T18:56:24.100Z 23 460-479 34 42 42 65 77 77 F F F <---
47372024-02-07T18:56:24.100Z 24 480-499 31 31 31 57 57 57 F F F
47382024-02-07T18:56:24.100Z 25 500-519 32 42 42 59 77 77 F T T <---
47392024-02-07T18:56:24.100Z 26 520-539 40 42 42 75 77 77 T F F <---
47402024-02-07T18:56:24.100Z 27 540-559 40 42 42 75 77 77 F F F <---
47412024-02-07T18:56:24.100Z 28 560-579 40 40 40 74 74 74 F F F
47422024-02-07T18:56:24.100Z 29 580-599 34 34 34 62 62 62 F F F
47432024-02-07T18:56:24.100ZMax gen: 42, Max flush: 78
47442024-02-07T18:56:24.100ZError: Difference in extent metadata found!
47452024-02-07T18:56:24.100ZA Difference in extent metadata is expected here
47462024-02-07T18:56:24.100ZOn loop 21
47472024-02-07T18:56:24.100Z
47482024-02-07T18:56:24.100ZVerifying data now
47492024-02-07T18:56:29.332ZNo change after verify
47502024-02-07T18:56:29.332ZLoop: 21 Downstairs dump after verify (and repair):
47512024-02-07T18:56:29.333ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47522024-02-07T18:56:29.333Z 0 000-019 43 43 43 79 79 79 F F F
47532024-02-07T18:56:29.333Z 1 020-039 42 42 42 78 78 78 F F F
47542024-02-07T18:56:29.333Z 2 040-059 30 30 30 56 56 56 F F F
47552024-02-07T18:56:29.333Z 3 060-079 38 38 38 71 71 71 F F F
47562024-02-07T18:56:29.333Z 4 080-099 32 32 32 59 59 59 F F F
47572024-02-07T18:56:29.333Z 5 100-119 40 40 40 74 74 74 F F F
47582024-02-07T18:56:29.333Z 6 120-139 43 43 43 79 79 79 F F F
47592024-02-07T18:56:29.333Z 7 140-159 43 43 43 79 79 79 F F F
47602024-02-07T18:56:29.333Z 8 160-179 40 40 40 75 75 75 F F F
47612024-02-07T18:56:29.333Z 9 180-199 33 33 33 61 61 61 F F F
47622024-02-07T18:56:29.333Z 10 200-219 40 40 40 75 75 75 F F F
47632024-02-07T18:56:29.333Z 11 220-239 42 42 42 77 77 77 F F F
47642024-02-07T18:56:29.333Z 12 240-259 41 41 41 76 76 76 F F F
47652024-02-07T18:56:29.333Z 13 260-279 41 41 41 76 76 76 F F F
47662024-02-07T18:56:29.333Z 14 280-299 40 40 40 75 75 75 F F F
47672024-02-07T18:56:29.333Z 15 300-319 43 43 43 79 79 79 F F F
47682024-02-07T18:56:29.333Z 16 320-339 41 41 41 76 76 76 F F F
47692024-02-07T18:56:29.333Z 17 340-359 41 41 41 76 76 76 F F F
47702024-02-07T18:56:29.333Z 18 360-379 40 40 40 75 75 75 F F F
47712024-02-07T18:56:29.334Z 19 380-399 40 40 40 74 74 74 F F F
47722024-02-07T18:56:29.334Z 20 400-419 34 34 34 64 64 64 F F F
47732024-02-07T18:56:29.334Z 21 420-439 36 36 36 67 67 67 F F F
47742024-02-07T18:56:29.334Z 22 440-459 37 37 37 70 70 70 F F F
47752024-02-07T18:56:29.334Z 23 460-479 42 42 42 77 77 77 F F F
47762024-02-07T18:56:29.334Z 24 480-499 31 31 31 57 57 57 F F F
47772024-02-07T18:56:29.334Z 25 500-519 43 43 43 79 79 79 F F F
47782024-02-07T18:56:29.334Z 26 520-539 42 42 42 77 77 77 F F F
47792024-02-07T18:56:29.334Z 27 540-559 42 42 42 77 77 77 F F F
47802024-02-07T18:56:29.334Z 28 560-579 40 40 40 74 74 74 F F F
47812024-02-07T18:56:29.372Z 29 580-599 34 34 34 62 62 62 F F F
47822024-02-07T18:56:29.372ZMax gen: 43, Max flush: 79
47832024-02-07T18:56:29.372Z
47842024-02-07T18:56:29.373ZBegin loop 22
47852024-02-07T18:56:29.373ZDownstairs to restart: 1
47862024-02-07T18:56:29.373Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.505536022Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311}
47872024-02-07T18:56:29.373Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.505812545Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311}
47882024-02-07T18:56:29.373Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.505853579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311}
47892024-02-07T18:56:29.373Z{Crucible runtime is spawned
47902024-02-07T18:56:29.373Z"msg":"Crucible 3b81689c-1d31-4f83-aaf2-cc40d0ee7ff5 has session id: c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.505907472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47912024-02-07T18:56:29.373Z{"msg":"Upstairs opts: Upstairs UUID: 3b81689c-1d31-4f83-aaf2-cc40d0ee7ff5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.505957185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47922024-02-07T18:56:29.373Z{"msg":"Crucible stats registered with UUID: 3b81689c-1d31-4f83-aaf2-cc40d0ee7ff5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.505987301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47932024-02-07T18:56:29.373Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506024396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311}
47942024-02-07T18:56:29.373Z{"msg":"3b81689c-1d31-4f83-aaf2-cc40d0ee7ff5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506060551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47952024-02-07T18:56:29.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506082528Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47962024-02-07T18:56:29.373Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.50613903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47972024-02-07T18:56:29.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506163547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47982024-02-07T18:56:29.373Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506186704Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
47992024-02-07T18:56:29.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506207981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48002024-02-07T18:56:29.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506248276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48012024-02-07T18:56:29.374Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506295129Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48022024-02-07T18:56:29.374Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506321556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48032024-02-07T18:56:29.374Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506353001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48042024-02-07T18:56:29.374Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506468806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48052024-02-07T18:56:29.374Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:27.506529278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48062024-02-07T18:56:29.374Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:27.506573972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48072024-02-07T18:56:29.374Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:27.506617026Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48082024-02-07T18:56:29.374Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506645212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48092024-02-07T18:56:29.374Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506687666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48102024-02-07T18:56:29.374Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506731041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48112024-02-07T18:56:29.374Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.506760557Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48122024-02-07T18:56:29.374Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.507217735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48132024-02-07T18:56:29.374Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.50725101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48142024-02-07T18:56:29.374Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.507368214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48152024-02-07T18:56:29.374Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.507394601Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48162024-02-07T18:56:29.374Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.507780539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48172024-02-07T18:56:29.375Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.50791583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48182024-02-07T18:56:29.375Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.508390556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48192024-02-07T18:56:29.375Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.50843431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48202024-02-07T18:56:29.375Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.508461456Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48212024-02-07T18:56:29.375Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.508486193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48222024-02-07T18:56:29.375Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.50851173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48232024-02-07T18:56:29.375Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:27.508536866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48242024-02-07T18:56:29.375Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.507959873Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48252024-02-07T18:56:29.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.508052071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48262024-02-07T18:56:29.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.508856322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48272024-02-07T18:56:29.375Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.508900346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48282024-02-07T18:56:29.375Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.509394879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48292024-02-07T18:56:29.375Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.509974281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48302024-02-07T18:56:29.375Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510035972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48312024-02-07T18:56:29.375Z{"msg":"[0]R flush_numbers[0..12]: [79, 78, 56, 71, 59, 74, 79, 79, 75, 61, 75, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510074347Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48322024-02-07T18:56:29.375Z{"msg":"[0]R generation[0..12]: [43, 42, 30, 38, 32, 40, 43, 43, 40, 33, 40, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510094804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48332024-02-07T18:56:29.375Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510121961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48342024-02-07T18:56:29.375Z{"msg":"[1]R flush_numbers[0..12]: [79, 78, 56, 71, 59, 74, 79, 79, 75, 61, 75, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510140788Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48352024-02-07T18:56:29.375Z{"msg":"[1]R generation[0..12]: [43, 42, 30, 38, 32, 40, 43, 43, 40, 33, 40, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510167195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"Wait for a query_work_queue command to finish before sending IO
48362024-02-07T18:56:29.375Zdownstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48372024-02-07T18:56:29.375Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510197581Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48382024-02-07T18:56:29.375Z{"msg":"[2]R flush_numbers[0..12]: [79, 78, 56, 71, 59, 74, 79, 79, 75, 61, 75, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510216678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48392024-02-07T18:56:29.375Z{"msg":"[2]R generation[0..12]: [43, 42, 30, 38, 32, 40, 43, 43, 40, 33, 40, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510241635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48402024-02-07T18:56:29.375Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510259842Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48412024-02-07T18:56:29.376Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510284609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48422024-02-07T18:56:29.376Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510302556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48432024-02-07T18:56:29.376Z{"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510326663Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48442024-02-07T18:56:29.376Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510343911Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48452024-02-07T18:56:29.376Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510368878Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48462024-02-07T18:56:29.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510384555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48472024-02-07T18:56:29.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510409032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48482024-02-07T18:56:29.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.51042585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48492024-02-07T18:56:29.376Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510450567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48502024-02-07T18:56:29.376Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510467844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48512024-02-07T18:56:29.376Z{"msg":"3b81689c-1d31-4f83-aaf2-cc40d0ee7ff5 is now active with session: c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510504589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48522024-02-07T18:56:29.376Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510530976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311,"session_id":"c9f13be9-ef8c-47b5-b3a4-5f71c59b18b9"}
48532024-02-07T18:56:29.376ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48542024-02-07T18:56:29.376Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:28.510578159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311}
48552024-02-07T18:56:29.376ZLoading write count information from file "/tmp/test_repair_verify.data"
48562024-02-07T18:56:29.376ZRun Repair workload
48572024-02-07T18:56:29.376Z01/30 Write block 510 len 12288 data: 6 4 6
48582024-02-07T18:56:29.376Z02/30 Read block 170 len 24576
48592024-02-07T18:56:29.376Z03/30 Read block 21 len 40960
48602024-02-07T18:56:29.376Z04/30 Read block 337 len 16384
48612024-02-07T18:56:29.376Z05/30 Write block 179 len 8192 data: 4 6
48622024-02-07T18:56:29.377Z06/30 Write block 48 len 32768 data: 3 4 4 7 6 7 7 7
48632024-02-07T18:56:29.377Z07/30 Write block 28 len 36864 data: 4 5 5 5 4 4 4 5 5
48642024-02-07T18:56:29.377Z08/30 Write block 46 len 12288 data: 3 3 4
48652024-02-07T18:56:29.377Z09/30 Read block 364 len 4096
48662024-02-07T18:56:29.377Z10/30 Read block 569 len 40960
48672024-02-07T18:56:29.377Z11/30 Write block 297 len 16384 data: 5 4 2 2
48682024-02-07T18:56:29.377Z12/30 Read block 225 len 40960
48692024-02-07T18:56:29.377Z13/30 Read block 139 len 32768
48702024-02-07T18:56:29.377Z14/30 Read block 567 len 4096
48712024-02-07T18:56:29.377Z15/30 Read block 326 len 8192
48722024-02-07T18:56:29.377Z16/30 Read block 274 len 12288
48732024-02-07T18:56:29.377Z17/30 Write block 162 len 8192 data: 6 6
48742024-02-07T18:56:29.377Z18/30 Write block 204 len 4096 data: 5
48752024-02-07T18:56:29.377Z19/30 Write block 205 len 36864 data: 6 5 4 6 4 3 4 4 5
48762024-02-07T18:56:29.377Z20/30 Write block 581 len 40960 data: 4 4 4 4 4 5 4 4 4 4
48772024-02-07T18:56:29.377Z21/30 Flush
48782024-02-07T18:56:29.377Z22/30 Write block 109 len 16384 data: 8 6 6 4
48792024-02-07T18:56:29.377Z23/30 Read block 265 len 36864
48802024-02-07T18:56:29.377Z24/30 Read block 402 len 4096
48812024-02-07T18:56:29.386Z25/30 Read block 375 len 24576
48822024-02-07T18:56:29.386Z26/30 Flush
48832024-02-07T18:56:29.386Z27/30 Write block 451 len 4096 data: 4
48842024-02-07T18:56:29.386Z28/30 Write block 578 len 20480 data: 3 3 3 5 5
48852024-02-07T18:56:29.386Z29/30 Read block 241 len 36864
48862024-02-07T18:56:29.386Z30/30 Read block 414 len 12288
48872024-02-07T18:56:29.386Z----------------------------------------------------------------
48882024-02-07T18:56:29.386Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48892024-02-07T18:56:29.386ZStates: Active Active Active
48902024-02-07T18:56:29.386ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48912024-02-07T18:56:29.387Z 1 Acked 1000 Write 3 Done Sent Done false
48922024-02-07T18:56:29.387Z 2 Acked 1001 Read 6 Done Sent Done false
48932024-02-07T18:56:29.387Z 3 Acked 1002 Read 10 Done Sent Done false
48942024-02-07T18:56:29.387Z 4 Acked 1003 Read 4 Done Sent Done false
48952024-02-07T18:56:29.387Z 5 Acked 1004 Write 2 Done Sent Done false
48962024-02-07T18:56:29.387Z 6 Acked 1005 Write 8 Done Sent Done false
48972024-02-07T18:56:29.387Z 7 Acked 1006 Write 9 Done Sent Done false
48982024-02-07T18:56:29.387Z 8 Acked 1007 Write 3 Done Sent Done false
48992024-02-07T18:56:29.387Z 9 Acked 1008 Read 1 Done Sent Done false
49002024-02-07T18:56:29.388Z 10 Acked 1009 Read 10 Done Sent Done false
49012024-02-07T18:56:29.388Z 11 Acked 1010 Write 4 Done Sent Done false
49022024-02-07T18:56:29.388Z 12 Acked 1011 Read 10 Done Sent Done false
49032024-02-07T18:56:29.388Z 13 Acked 1012 Read 8 Done Sent Done false
49042024-02-07T18:56:29.388Z 14 Acked 1013 Read 1 Done Sent Done false
49052024-02-07T18:56:29.388Z 15 Acked 1014 Read 2 Done Sent Done false
49062024-02-07T18:56:29.388Z 16 Acked 1015 Read 3 Done Sent Done false
49072024-02-07T18:56:29.388Z 17 Acked 1016 Write 2 Done Sent Done false
49082024-02-07T18:56:29.388Z 18 Acked 1017 Write 1 Done Sent Done false
49092024-02-07T18:56:29.388Z 19 Acked 1018 Write 9 Done Sent Done false
49102024-02-07T18:56:29.388Z 20 Acked 1019 Write 10 Done Sent Done false
49112024-02-07T18:56:29.388Z 21 Acked 1020 Flush 0 Done Sent Done false
49122024-02-07T18:56:29.388Z 22 Acked 1021 Write 4 Done Sent Done false
49132024-02-07T18:56:29.389Z 23 Acked 1022 Read 9 Done Sent Done false
49142024-02-07T18:56:29.389Z 24 Acked 1023 Read 1 Done Sent Done false
49152024-02-07T18:56:29.389Z 25 Acked 1024 Read 6 Done Sent Done false
49162024-02-07T18:56:29.389Z 26 Acked 1025 Flush 0 Done Sent Done false
49172024-02-07T18:56:29.389Z 27 Acked 1026 Write 1 Done Sent Done false
49182024-02-07T18:56:29.389Z 28 Acked 1027 Write 5 Done Sent Done false
49192024-02-07T18:56:29.389Z 29 Acked 1028 Read 9 Done Sent Done false
49202024-02-07T18:56:29.389Z 30 Acked 1029 Read 3 Done Sent Done false
49212024-02-07T18:56:29.389Z STATES DS:0 DS:1 DS:2 TOTAL
49222024-02-07T18:56:29.389Z New 0 0 0 0
49232024-02-07T18:56:29.389Z Sent 0 30 0 30
49242024-02-07T18:56:29.389Z Done 30 0 30 60
49252024-02-07T18:56:29.389Z Skipped 0 0 0 0
49262024-02-07T18:56:29.389Z Error 0 0 0 0
49272024-02-07T18:56:29.390ZLast Flush: 1025 0 1025
49282024-02-07T18:56:29.390ZDownstairs last five completed:
49292024-02-07T18:56:29.390ZUpstairs last five completed: 30 29 28 27 26
49302024-02-07T18:56:29.390Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:28.540383268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1311}
49312024-02-07T18:56:29.390ZWrote out file "/tmp/test_repair_verify.data"
49322024-02-07T18:56:29.390Z
49332024-02-07T18:56:29.390ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
49342024-02-07T18:56:29.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49352024-02-07T18:56:29.390Z 0 000-019 43 43 43 79 79 79 F F F
49362024-02-07T18:56:29.390Z 1 020-039 44 42 44 80 78 80 F F F <---
49372024-02-07T18:56:29.390Z 2 040-059 44 30 44 80 56 80 F F F <---
49382024-02-07T18:56:29.390Z 3 060-079 38 38 38 71 71 71 F F F
49392024-02-07T18:56:29.390Z 4 080-099 32 32 32 59 59 59 F F F
49402024-02-07T18:56:29.391Z 5 100-119 44 40 44 81 74 81 F F F <---
49412024-02-07T18:56:29.391Z 6 120-139 43 43 43 79 79 79 F F F
49422024-02-07T18:56:29.391Z 7 140-159 43 43 43 79 79 79 F F F
49432024-02-07T18:56:29.391Z 8 160-179 44 40 44 80 75 80 F F F <---
49442024-02-07T18:56:29.391Z 9 180-199 44 33 44 80 61 80 F F F <---
49452024-02-07T18:56:29.391Z 10 200-219 44 40 44 80 75 80 F F F <---
49462024-02-07T18:56:29.391Z 11 220-239 42 42 42 77 77 77 F F F
49472024-02-07T18:56:29.391Z 12 240-259 41 41 41 76 76 76 F F F
49482024-02-07T18:56:29.391Z 13 260-279 41 41 41 76 76 76 F F F
49492024-02-07T18:56:29.391Z 14 280-299 44 40 44 80 75 80 F F F <---
49502024-02-07T18:56:29.391Z 15 300-319 44 43 44 80 79 80 F F F <---
49512024-02-07T18:56:29.391Z 16 320-339 41 41 41 76 76 76 F F F
49522024-02-07T18:56:29.392Z 17 340-359 41 41 41 76 76 76 F F F
49532024-02-07T18:56:29.392Z 18 360-379 40 40 40 75 75 75 F F F
49542024-02-07T18:56:29.392Z 19 380-399 40 40 40 74 74 74 F F F
49552024-02-07T18:56:29.392Z 20 400-419 34 34 34 64 64 64 F F F
49562024-02-07T18:56:29.392Z 21 420-439 36 36 36 67 67 67 F F F
49572024-02-07T18:56:29.392Z 22 440-459 37 37 37 70 70 70 T F T <---
49582024-02-07T18:56:29.392Z 23 460-479 42 42 42 77 77 77 F F F
49592024-02-07T18:56:29.392Z 24 480-499 31 31 31 57 57 57 F F F
49602024-02-07T18:56:29.392Z 25 500-519 44 43 44 80 79 80 F F F <---
49612024-02-07T18:56:29.392Z 26 520-539 42 42 42 77 77 77 F F F
49622024-02-07T18:56:29.392Z 27 540-559 42 42 42 77 77 77 F F F
49632024-02-07T18:56:29.392Z 28 560-579 40 40 40 74 74 74 T F T <---
49642024-02-07T18:56:29.393Z 29 580-599 44 34 44 80 62 80 T F T <---
49652024-02-07T18:56:29.393ZMax gen: 44, Max flush: 81
49662024-02-07T18:56:29.393ZError: Difference in extent metadata found!
49672024-02-07T18:56:29.393ZA Difference in extent metadata is expected here
49682024-02-07T18:56:29.393ZOn loop 22
49692024-02-07T18:56:29.393Z
49702024-02-07T18:56:29.393ZVerifying data now
49712024-02-07T18:56:33.918ZNo change after verify
49722024-02-07T18:56:33.918ZLoop: 22 Downstairs dump after verify (and repair):
49732024-02-07T18:56:33.934ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49742024-02-07T18:56:33.934Z 0 000-019 43 43 43 79 79 79 F F F
49752024-02-07T18:56:33.934Z 1 020-039 44 44 44 80 80 80 F F F
49762024-02-07T18:56:33.934Z 2 040-059 44 44 44 80 80 80 F F F
49772024-02-07T18:56:33.934Z 3 060-079 38 38 38 71 71 71 F F F
49782024-02-07T18:56:33.934Z 4 080-099 32 32 32 59 59 59 F F F
49792024-02-07T18:56:33.934Z 5 100-119 44 44 44 81 81 81 F F F
49802024-02-07T18:56:33.934Z 6 120-139 43 43 43 79 79 79 F F F
49812024-02-07T18:56:33.937Z 7 140-159 43 43 43 79 79 79 F F F
49822024-02-07T18:56:33.937Z 8 160-179 44 44 44 80 80 80 F F F
49832024-02-07T18:56:33.937Z 9 180-199 44 44 44 80 80 80 F F F
49842024-02-07T18:56:33.937Z 10 200-219 44 44 44 80 80 80 F F F
49852024-02-07T18:56:33.937Z 11 220-239 42 42 42 77 77 77 F F F
49862024-02-07T18:56:33.937Z 12 240-259 41 41 41 76 76 76 F F F
49872024-02-07T18:56:33.937Z 13 260-279 41 41 41 76 76 76 F F F
49882024-02-07T18:56:33.938Z 14 280-299 44 44 44 80 80 80 F F F
49892024-02-07T18:56:33.938Z 15 300-319 44 44 44 80 80 80 F F F
49902024-02-07T18:56:33.938Z 16 320-339 41 41 41 76 76 76 F F F
49912024-02-07T18:56:33.938Z 17 340-359 41 41 41 76 76 76 F F F
49922024-02-07T18:56:33.938Z 18 360-379 40 40 40 75 75 75 F F F
49932024-02-07T18:56:33.938Z 19 380-399 40 40 40 74 74 74 F F F
49942024-02-07T18:56:33.938Z 20 400-419 34 34 34 64 64 64 F F F
49952024-02-07T18:56:33.938Z 21 420-439 36 36 36 67 67 67 F F F
49962024-02-07T18:56:33.938Z 22 440-459 45 45 45 82 82 82 F F F
49972024-02-07T18:56:33.938Z 23 460-479 42 42 42 77 77 77 F F F
49982024-02-07T18:56:33.938Z 24 480-499 31 31 31 57 57 57 F F F
49992024-02-07T18:56:33.938Z 25 500-519 44 44 44 80 80 80 F F F
50002024-02-07T18:56:33.938Z 26 520-539 42 42 42 77 77 77 F F F
50012024-02-07T18:56:33.938Z 27 540-559 42 42 42 77 77 77 F F F
50022024-02-07T18:56:33.938Z 28 560-579 45 45 45 82 82 82 F F F
50032024-02-07T18:56:33.938Z 29 580-599 45 45 45 82 82 82 F F F
50042024-02-07T18:56:33.938ZMax gen: 45, Max flush: 82
50052024-02-07T18:56:33.938Z
50062024-02-07T18:56:33.938ZBegin loop 23
50072024-02-07T18:56:33.938ZDownstairs to restart: 2
50082024-02-07T18:56:33.957Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.214555813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319}
50092024-02-07T18:56:33.957Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
50102024-02-07T18:56:33.957Z,"time":"2024-02-07T18:56:33.214894537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319}
50112024-02-07T18:56:33.962Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.214938712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319}
50122024-02-07T18:56:33.963Z{"msg":"Crucible f914f54c-639e-43ac-9d1d-33199e108cd9 has session id: 591af89b-d4f3-4a3b-884f-21c5d3eed4c3","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.214966788Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50132024-02-07T18:56:33.963Z{"msg":"Upstairs opts: Upstairs UUID: f914f54c-639e-43ac-9d1d-33199e108cd9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215010512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50142024-02-07T18:56:33.963Z{"msg":"Crucible stats registered with UUID: f914f54c-639e-43ac-9d1d-33199e108cd9","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215056676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50152024-02-07T18:56:33.963Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.21509971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319}
50162024-02-07T18:56:33.963Z{"msg":"f914f54c-639e-43ac-9d1d-33199e108cd9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215241301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50172024-02-07T18:56:33.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215278456Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50182024-02-07T18:56:33.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215298593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50192024-02-07T18:56:33.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215330459Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50202024-02-07T18:56:33.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215346067Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50212024-02-07T18:56:33.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215361674Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50222024-02-07T18:56:33.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.21539688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50232024-02-07T18:56:33.963Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215416457Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50242024-02-07T18:56:33.964Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215441004Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50252024-02-07T18:56:33.964Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215482308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50262024-02-07T18:56:33.964Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215509724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50272024-02-07T18:56:33.964Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:33.215538781Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50282024-02-07T18:56:33.964Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215564627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50292024-02-07T18:56:33.964Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:33.215590923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50302024-02-07T18:56:33.964Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:33.215621629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50312024-02-07T18:56:33.964Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215645986Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50322024-02-07T18:56:33.964Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215668703Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50332024-02-07T18:56:33.964Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.215755161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50342024-02-07T18:56:33.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.216247745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50352024-02-07T18:56:33.964Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.216286219Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50362024-02-07T18:56:33.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.216302937Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50372024-02-07T18:56:33.965Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.216317015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50382024-02-07T18:56:33.965Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.216803689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50392024-02-07T18:56:33.965Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.216897817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50402024-02-07T18:56:33.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.21738942Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50412024-02-07T18:56:33.965Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.217449892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50422024-02-07T18:56:33.965Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.217486407Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50432024-02-07T18:56:33.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.217502555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50442024-02-07T18:56:33.965Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.217524282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50452024-02-07T18:56:33.965Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:33.21753819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50462024-02-07T18:56:34.959Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.217188847Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50472024-02-07T18:56:34.960Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.217338676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50482024-02-07T18:56:34.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.218204479Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50492024-02-07T18:56:34.964Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.218241624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50502024-02-07T18:56:34.964Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.218758555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50512024-02-07T18:56:34.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219322808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50522024-02-07T18:56:34.965Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219364882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50532024-02-07T18:56:34.965Z{"msg":"[0]R flush_numbers[0..12]: [79, 80, 80, 71, 59, 81, 79, 79, 80, 80, 80, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.21938249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50542024-02-07T18:56:34.965Z{"msg":"[0]R generation[0..12]: [43, 44, 44, 38, 32, 44, 43, 43, 44, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219397478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50552024-02-07T18:56:34.965Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219413106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50562024-02-07T18:56:34.965Z{"msg":"[1]R flush_numbers[0..12]: [79, 80, 80, 71, 59, 81, 79, 79, 80, 80, 80, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219435373Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50572024-02-07T18:56:34.965Z{"msg":"[1]R generation[0..12]: [43, 44, 44, 38, 32, 44, 43, 43, 44, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219449961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50582024-02-07T18:56:34.965Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219464819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50592024-02-07T18:56:34.965Z{"msg":"[2]R flush_numbers[0..12]: [79, 80, 80, 71, 59, 81, 79, 79, 80, 80, 80, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219485626Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50602024-02-07T18:56:34.965Z{"msg":"[2]R generation[0..12]: [43, 44, 44, 38, 32, 44, 43, 43, 44, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219499884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50612024-02-07T18:56:34.965Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219514002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50622024-02-07T18:56:34.965Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.21952797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50632024-02-07T18:56:34.965Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219542738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50642024-02-07T18:56:34.965Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219563446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50652024-02-07T18:56:34.965ZWait for a query_work_queue command to finish before sending IO
50662024-02-07T18:56:34.965Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219584673Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50672024-02-07T18:56:34.965Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.21960549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50682024-02-07T18:56:34.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219618558Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50692024-02-07T18:56:34.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219638356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50702024-02-07T18:56:34.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219652634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50712024-02-07T18:56:34.965Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219666002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50722024-02-07T18:56:34.965Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219685979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50732024-02-07T18:56:34.965Z{"msg":"f914f54c-639e-43ac-9d1d-33199e108cd9 is now active with session: 591af89b-d4f3-4a3b-884f-21c5d3eed4c3","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219699537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50742024-02-07T18:56:34.966Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219712575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319,"session_id":"591af89b-d4f3-4a3b-884f-21c5d3eed4c3"}
50752024-02-07T18:56:34.966Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:34.219732023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319}
50762024-02-07T18:56:34.966ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50772024-02-07T18:56:34.966ZLoading write count information from file "/tmp/test_repair_verify.data"
50782024-02-07T18:56:34.966ZRun Repair workload
50792024-02-07T18:56:34.969Z01/30 Write block 467 len 16384 data: 3 2 2 2
50802024-02-07T18:56:34.969Z02/30 Write block 570 len 16384 data: 3 3 2 3
50812024-02-07T18:56:34.969Z03/30 Read block 104 len 8192
50822024-02-07T18:56:34.969Z04/30 Write block 16 len 28672 data: 6 7 6 9 9 8 9
50832024-02-07T18:56:34.969Z05/30 Write block 368 len 12288 data: 5 4 4
50842024-02-07T18:56:34.969Z06/30 Write block 347 len 20480 data: 6 6 5 6 6
50852024-02-07T18:56:34.969Z07/30 Write block 231 len 8192 data: 5 6
50862024-02-07T18:56:34.969Z08/30 Read block 182 len 12288
50872024-02-07T18:56:34.972Z09/30 Write block 320 len 24576 data: 6 5 5 5 4 3
50882024-02-07T18:56:34.972Z10/30 Read block 294 len 20480
50892024-02-07T18:56:34.973Z11/30 Read block 510 len 40960
50902024-02-07T18:56:34.973Z12/30 Read block 169 len 28672
50912024-02-07T18:56:34.976Z13/30 Read block 176 len 36864
50922024-02-07T18:56:34.976Z14/30 Write block 458 len 36864 data: 3 3 2 3 3 4 4 4 4
50932024-02-07T18:56:34.977Z15/30 Read block 226 len 28672
50942024-02-07T18:56:34.980Z16/30 Write block 537 len 32768 data: 6 6 6 6 6 6 3 3
50952024-02-07T18:56:34.980Z17/30 Write block 153 len 16384 data: 7 6 3 2
50962024-02-07T18:56:34.980Z18/30 Read block 136 len 36864
50972024-02-07T18:56:34.980Z19/30 Write block 232 len 24576 data: 7 7 8 11 9 8
50982024-02-07T18:56:34.983Z20/30 Write block 378 len 4096 data: 4
50992024-02-07T18:56:34.983Z21/30 Write block 425 len 32768 data: 5 5 5 7 6 5 5 3
51002024-02-07T18:56:34.983Z22/30 Write block 494 len 4096 data: 7
51012024-02-07T18:56:34.983Z23/30 Write block 315 len 20480 data: 5 7 5 6 5
51022024-02-07T18:56:34.983Z24/30 Write block 115 len 36864 data: 3 3 4 6 7 6 6 6 4
51032024-02-07T18:56:34.986Z25/30 Read block 206 len 24576
51042024-02-07T18:56:34.986Z26/30 Read block 319 len 40960
51052024-02-07T18:56:34.986Z27/30 Read block 309 len 36864
51062024-02-07T18:56:34.989Z28/30 Write block 338 len 40960 data: 6 8 6 4 4 4 5 7 8 7
51072024-02-07T18:56:34.989Z29/30 Write block 550 len 24576 data: 4 4 4 4 4 4
51082024-02-07T18:56:34.989Z30/30 Write block 125 len 24576 data: 4 4 4 5 5 4
51092024-02-07T18:56:34.992Z----------------------------------------------------------------
51102024-02-07T18:56:34.992Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
51112024-02-07T18:56:34.992ZStates: Active Active Active
51122024-02-07T18:56:34.992ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
51132024-02-07T18:56:34.993Z 1 Acked 1000 Write 4 Done Done Done false
51142024-02-07T18:56:34.993Z 2 Acked 1001 Write 4 Done Done Done false
51152024-02-07T18:56:34.993Z 3 Acked 1002 Read 2 Done Done Done false
51162024-02-07T18:56:34.993Z 4 Acked 1003 Write 7 Done Done Done false
51172024-02-07T18:56:34.993Z 5 Acked 1004 Write 3 Done Done Sent false
51182024-02-07T18:56:34.993Z 6 Acked 1005 Write 5 Done Done Sent false
51192024-02-07T18:56:34.993Z 7 Acked 1006 Write 2 Done Done Sent false
51202024-02-07T18:56:34.993Z 8 Acked 1007 Read 3 Done Done Sent false
51212024-02-07T18:56:34.993Z 9 Acked 1008 Write 6 Done Done Sent false
51222024-02-07T18:56:34.993Z 10 Acked 1009 Read 5 Done Done Sent false
51232024-02-07T18:56:34.993Z 11 Acked 1010 Read 10 Done Done Sent false
51242024-02-07T18:56:34.993Z 12 Acked 1011 Read 7 Done Done Sent false
51252024-02-07T18:56:34.993Z 13 Acked 1012 Read 9 Done Done Sent false
51262024-02-07T18:56:34.993Z 14 Acked 1013 Write 9 Done Done Sent false
51272024-02-07T18:56:34.993Z 15 Acked 1014 Read 7 Done Done Sent false
51282024-02-07T18:56:34.993Z 16 Acked 1015 Write 8 Done Done Sent false
51292024-02-07T18:56:34.993Z 17 Acked 1016 Write 4 Done Done Sent false
51302024-02-07T18:56:34.993Z 18 Acked 1017 Read 9 Done Done Sent false
51312024-02-07T18:56:34.993Z 19 Acked 1018 Write 6 Done Done Sent false
51322024-02-07T18:56:34.993Z 20 Acked 1019 Write 1 Done Done Sent false
51332024-02-07T18:56:34.993Z 21 Acked 1020 Write 8 Done Done Sent false
51342024-02-07T18:56:34.993Z 22 Acked 1021 Write 1 Done Done Sent false
51352024-02-07T18:56:34.993Z 23 Acked 1022 Write 5 Done Done Sent false
51362024-02-07T18:56:34.993Z 24 Acked 1023 Write 9 Done Done Sent false
51372024-02-07T18:56:34.994Z 25 Acked 1024 Read 6 Done Done Sent false
51382024-02-07T18:56:34.994Z 26 Acked 1025 Read 10 Done Done Sent false
51392024-02-07T18:56:34.994Z 27 Acked 1026 Read 9 Done Done Sent false
51402024-02-07T18:56:34.994Z 28 Acked 1027 Write 10 Done Done Sent false
51412024-02-07T18:56:34.994Z 29 Acked 1028 Write 6 Done Done Sent false
51422024-02-07T18:56:34.994Z 30 Acked 1029 Write 6 Sent Sent Sent false
51432024-02-07T18:56:34.994Z STATES DS:0 DS:1 DS:2 TOTAL
51442024-02-07T18:56:34.994Z New 0 0 0 0
51452024-02-07T18:56:34.994Z Sent 1 1 26 28
51462024-02-07T18:56:34.994Z Done 29 29 4 62
51472024-02-07T18:56:34.994Z Skipped 0 0 0 0
51482024-02-07T18:56:34.994Z Error 0 0 0 0
51492024-02-07T18:56:34.994ZLast Flush: 0 0 0
51502024-02-07T18:56:34.994ZDownstairs last five completed:
51512024-02-07T18:56:34.994ZUpstairs last five completed: 30 29 28 27 26
51522024-02-07T18:56:34.994Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:34.247830683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1319}
51532024-02-07T18:56:34.994ZWrote out file "/tmp/test_repair_verify.data"
51542024-02-07T18:56:34.997Z
51552024-02-07T18:56:35.001ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51562024-02-07T18:56:35.021ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51572024-02-07T18:56:35.021Z 0 000-019 43 43 43 79 79 79 T T T
51582024-02-07T18:56:35.021Z 1 020-039 44 44 44 80 80 80 T T T
51592024-02-07T18:56:35.021Z 2 040-059 44 44 44 80 80 80 F F F
51602024-02-07T18:56:35.021Z 3 060-079 38 38 38 71 71 71 F F F
51612024-02-07T18:56:35.021Z 4 080-099 32 32 32 59 59 59 F F F
51622024-02-07T18:56:35.021Z 5 100-119 44 44 44 81 81 81 T T F <---
51632024-02-07T18:56:35.021Z 6 120-139 43 43 43 79 79 79 T T F <---
51642024-02-07T18:56:35.021Z 7 140-159 43 43 43 79 79 79 T T F <---
51652024-02-07T18:56:35.021Z 8 160-179 44 44 44 80 80 80 F F F
51662024-02-07T18:56:35.021Z 9 180-199 44 44 44 80 80 80 F F F
51672024-02-07T18:56:35.021Z 10 200-219 44 44 44 80 80 80 F F F
51682024-02-07T18:56:35.021Z 11 220-239 42 42 42 77 77 77 T T F <---
51692024-02-07T18:56:35.021Z 12 240-259 41 41 41 76 76 76 F F F
51702024-02-07T18:56:35.021Z 13 260-279 41 41 41 76 76 76 F F F
51712024-02-07T18:56:35.021Z 14 280-299 44 44 44 80 80 80 F F F
51722024-02-07T18:56:35.021Z 15 300-319 44 44 44 80 80 80 T T F <---
51732024-02-07T18:56:35.025Z 16 320-339 41 41 41 76 76 76 T T F <---
51742024-02-07T18:56:35.025Z 17 340-359 41 41 41 76 76 76 T T F <---
51752024-02-07T18:56:35.025Z 18 360-379 40 40 40 75 75 75 T T F <---
51762024-02-07T18:56:35.025Z 19 380-399 40 40 40 74 74 74 F F F
51772024-02-07T18:56:35.025Z 20 400-419 34 34 34 64 64 64 F F F
51782024-02-07T18:56:35.025Z 21 420-439 36 36 36 67 67 67 T T F <---
51792024-02-07T18:56:35.025Z 22 440-459 45 45 45 82 82 82 T T F <---
51802024-02-07T18:56:35.025Z 23 460-479 42 42 42 77 77 77 T T T
51812024-02-07T18:56:35.025Z 24 480-499 31 31 31 57 57 57 T T F <---
51822024-02-07T18:56:35.025Z 25 500-519 44 44 44 80 80 80 F F F
51832024-02-07T18:56:35.025Z 26 520-539 42 42 42 77 77 77 T T F <---
51842024-02-07T18:56:35.025Z 27 540-559 42 42 42 77 77 77 T T F <---
51852024-02-07T18:56:35.025Z 28 560-579 45 45 45 82 82 82 T T T
51862024-02-07T18:56:35.025Z 29 580-599 45 45 45 82 82 82 F F F
51872024-02-07T18:56:35.026ZMax gen: 45, Max flush: 82
51882024-02-07T18:56:35.026ZError: Difference in extent metadata found!
51892024-02-07T18:56:35.026ZA Difference in extent metadata is expected here
51902024-02-07T18:56:35.026ZOn loop 23
51912024-02-07T18:56:35.026Z
51922024-02-07T18:56:35.029ZVerifying data now
51932024-02-07T18:56:40.861ZNo change after verify
51942024-02-07T18:56:40.862ZLoop: 23 Downstairs dump after verify (and repair):
51952024-02-07T18:56:40.875ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51962024-02-07T18:56:40.875Z 0 000-019 46 46 46 83 83 83 F F F
51972024-02-07T18:56:40.875Z 1 020-039 46 46 46 83 83 83 F F F
51982024-02-07T18:56:40.875Z 2 040-059 44 44 44 80 80 80 F F F
51992024-02-07T18:56:40.875Z 3 060-079 38 38 38 71 71 71 F F F
52002024-02-07T18:56:40.875Z 4 080-099 32 32 32 59 59 59 F F F
52012024-02-07T18:56:40.875Z 5 100-119 46 46 46 83 83 83 F F F
52022024-02-07T18:56:40.875Z 6 120-139 46 46 46 83 83 83 F F F
52032024-02-07T18:56:40.875Z 7 140-159 46 46 46 83 83 83 F F F
52042024-02-07T18:56:40.875Z 8 160-179 44 44 44 80 80 80 F F F
52052024-02-07T18:56:40.875Z 9 180-199 44 44 44 80 80 80 F F F
52062024-02-07T18:56:40.875Z 10 200-219 44 44 44 80 80 80 F F F
52072024-02-07T18:56:40.878Z 11 220-239 46 46 46 83 83 83 F F F
52082024-02-07T18:56:40.878Z 12 240-259 41 41 41 76 76 76 F F F
52092024-02-07T18:56:40.878Z 13 260-279 41 41 41 76 76 76 F F F
52102024-02-07T18:56:40.879Z 14 280-299 44 44 44 80 80 80 F F F
52112024-02-07T18:56:40.879Z 15 300-319 46 46 46 83 83 83 F F F
52122024-02-07T18:56:40.879Z 16 320-339 46 46 46 83 83 83 F F F
52132024-02-07T18:56:40.879Z 17 340-359 46 46 46 83 83 83 F F F
52142024-02-07T18:56:40.879Z 18 360-379 46 46 46 83 83 83 F F F
52152024-02-07T18:56:40.879Z 19 380-399 40 40 40 74 74 74 F F F
52162024-02-07T18:56:40.879Z 20 400-419 34 34 34 64 64 64 F F F
52172024-02-07T18:56:40.879Z 21 420-439 46 46 46 83 83 83 F F F
52182024-02-07T18:56:40.879Z 22 440-459 46 46 46 83 83 83 F F F
52192024-02-07T18:56:40.879Z 23 460-479 46 46 46 83 83 83 F F F
52202024-02-07T18:56:40.879Z 24 480-499 46 46 46 83 83 83 F F F
52212024-02-07T18:56:40.879Z 25 500-519 44 44 44 80 80 80 F F F
52222024-02-07T18:56:40.879Z 26 520-539 46 46 46 83 83 83 F F F
52232024-02-07T18:56:40.879Z 27 540-559 46 46 46 83 83 83 F F F
52242024-02-07T18:56:40.879Z 28 560-579 46 46 46 83 83 83 F F F
52252024-02-07T18:56:40.879Z 29 580-599 45 45 45 82 82 82 F F F
52262024-02-07T18:56:40.879ZMax gen: 46, Max flush: 83
52272024-02-07T18:56:40.882Z
52282024-02-07T18:56:40.882ZBegin loop 24
52292024-02-07T18:56:40.882ZDownstairs to restart: 2
52302024-02-07T18:56:40.899Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155278836Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327}
52312024-02-07T18:56:40.899Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\"Crucible runtime is spawned
52322024-02-07T18:56:40.899Z,\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155674743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327}
52332024-02-07T18:56:40.899Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155704799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327}
52342024-02-07T18:56:40.904Z{"msg":"Crucible ef0d6cdc-cf97-40d1-8725-f91086457d75 has session id: c5b34832-9207-4421-9553-10369e3c2a0b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155749453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52352024-02-07T18:56:40.904Z{"msg":"Upstairs opts: Upstairs UUID: ef0d6cdc-cf97-40d1-8725-f91086457d75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155800535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52362024-02-07T18:56:40.904Z{"msg":"Crucible stats registered with UUID: ef0d6cdc-cf97-40d1-8725-f91086457d75","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155818873Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52372024-02-07T18:56:40.904Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.15584053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327}
52382024-02-07T18:56:40.904Z{"msg":"ef0d6cdc-cf97-40d1-8725-f91086457d75 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155900272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52392024-02-07T18:56:40.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155927329Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52402024-02-07T18:56:40.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155942556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52412024-02-07T18:56:40.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.155963873Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52422024-02-07T18:56:40.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.15599018Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52432024-02-07T18:56:40.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156017266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52442024-02-07T18:56:40.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156043423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52452024-02-07T18:56:40.904Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156081448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52462024-02-07T18:56:40.904Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156120022Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52472024-02-07T18:56:40.904Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156147239Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52482024-02-07T18:56:40.904Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:40.156185634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52492024-02-07T18:56:40.904Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:40.156302718Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52502024-02-07T18:56:40.904Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:40.156343612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52512024-02-07T18:56:40.905Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156379887Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52522024-02-07T18:56:40.905Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156406144Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52532024-02-07T18:56:40.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.15643226Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52542024-02-07T18:56:40.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156475704Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52552024-02-07T18:56:40.905Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.156498141Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52562024-02-07T18:56:40.905Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.157062365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52572024-02-07T18:56:40.905Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.157108239Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52582024-02-07T18:56:40.905Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.157127396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52592024-02-07T18:56:40.905Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.157150403Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52602024-02-07T18:56:40.905Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.157723966Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52612024-02-07T18:56:40.905Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.157899212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52622024-02-07T18:56:40.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.158219978Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52632024-02-07T18:56:40.905Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.158260123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52642024-02-07T18:56:40.905Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.158286829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52652024-02-07T18:56:40.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.158474224Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52662024-02-07T18:56:40.905Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.158524787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52672024-02-07T18:56:40.905Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:40.158559322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52682024-02-07T18:56:41.903Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.158319564Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52692024-02-07T18:56:41.908Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.158456886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52702024-02-07T18:56:41.908Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.159248549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52712024-02-07T18:56:41.908Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.159306741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52722024-02-07T18:56:41.908Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.159813562Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52732024-02-07T18:56:41.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160384285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52742024-02-07T18:56:41.908Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160441157Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52752024-02-07T18:56:41.908Z{"msg":"[0]R flush_numbers[0..12]: [83, 83, 80, 71, 59, 83, 83, 83, 80, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160464804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52762024-02-07T18:56:41.908Z{"msg":"[0]R generation[0..12]: [46, 46, 44, 38, 32, 46, 46, 46, 44, 44, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.16049443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52772024-02-07T18:56:41.908Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160514098Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52782024-02-07T18:56:41.908Z{"msg":"[1]R flush_numbers[0..12]: [83, 83, 80, 71, 59, 83, 83, 83, 80, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160541174Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52792024-02-07T18:56:41.908Z{"msg":"[1]R generation[0..12]: [46, 46, 44, 38, 32, 46, 46, 46, 44, 44, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160559332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52802024-02-07T18:56:41.908Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160577839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52812024-02-07T18:56:41.908ZWait for a query_work_queue command to finish before sending IO
52822024-02-07T18:56:41.908Z{"msg":"[2]R flush_numbers[0..12]: [83, 83, 80, 71, 59, 83, 83, 83, 80, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160608615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52832024-02-07T18:56:41.908Z{"msg":"[2]R generation[0..12]: [46, 46, 44, 38, 32, 46, 46, 46, 44, 44, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160626892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52842024-02-07T18:56:41.908Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160652729Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52852024-02-07T18:56:41.908Z{"msg":"Max found gen is 47","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160675766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52862024-02-07T18:56:41.908Z{"msg":"Generation requested: 48 >= found:47","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.1607181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52872024-02-07T18:56:41.908Z{"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160740747Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52882024-02-07T18:56:41.908Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160782851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52892024-02-07T18:56:41.908Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160827555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52902024-02-07T18:56:41.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160855311Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52912024-02-07T18:56:41.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160885027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52922024-02-07T18:56:41.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160925492Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52932024-02-07T18:56:41.909Zhostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52942024-02-07T18:56:41.909Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.160973795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52952024-02-07T18:56:41.909Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.161007331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52962024-02-07T18:56:41.909Z{"msg":"ef0d6cdc-cf97-40d1-8725-f91086457d75 is now active with session: c5b34832-9207-4421-9553-10369e3c2a0b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.161025058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52972024-02-07T18:56:41.909Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.161048595Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327,"session_id":"c5b34832-9207-4421-9553-10369e3c2a0b"}
52982024-02-07T18:56:41.909Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:41.161064553Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327}
52992024-02-07T18:56:41.909ZLoading write count information from file "/tmp/test_repair_verify.data"
53002024-02-07T18:56:41.909ZRun Repair workload
53012024-02-07T18:56:41.909Z01/30 Write block 284 len 32768 data: 5 5 6 6 6 6 7 5
53022024-02-07T18:56:41.912Z02/30 Write block 410 len 8192 data: 4 5
53032024-02-07T18:56:41.912Z03/30 Write block 528 len 40960 data: 4 3 3 3 4 4 4 5 5 7
53042024-02-07T18:56:41.912Z04/30 Flush
53052024-02-07T18:56:41.912Z05/30 Flush
53062024-02-07T18:56:41.912Z06/30 Write block 252 len 36864 data: 3 2 2 3 5 4 4 7 7
53072024-02-07T18:56:41.912Z07/30 Write block 149 len 4096 data: 8
53082024-02-07T18:56:41.912Z08/30 Write block 122 len 12288 data: 7 5 4
53092024-02-07T18:56:41.915Z09/30 Write block 205 len 40960 data: 7 6 5 7 5 4 5 5 6 5
53102024-02-07T18:56:41.915Z10/30 Write block 71 len 20480 data: 7 5 5 3 3
53112024-02-07T18:56:41.915Z11/30 Write block 163 len 12288 data: 7 7 7
53122024-02-07T18:56:41.915Z12/30 Read block 218 len 16384
53132024-02-07T18:56:41.918Z13/30 Write block 105 len 16384 data: 6 6 6 6
53142024-02-07T18:56:41.918Z14/30 Write block 209 len 8192 data: 6 5
53152024-02-07T18:56:41.918Z15/30 Read block 436 len 40960
53162024-02-07T18:56:41.921Z16/30 Write block 268 len 24576 data: 2 2 2 3 3 3
53172024-02-07T18:56:41.921Z17/30 Write block 29 len 24576 data: 6 6 6 5 5 5
53182024-02-07T18:56:41.922Z18/30 Write block 566 len 36864 data: 3 3 3 3 4 4 3 4 3
53192024-02-07T18:56:41.922Z19/30 Read block 253 len 28672
53202024-02-07T18:56:41.924Z20/30 Read block 485 len 40960
53212024-02-07T18:56:41.924Z21/30 Read block 192 len 36864
53222024-02-07T18:56:41.927Z22/30 Write block 437 len 12288 data: 7 5 6
53232024-02-07T18:56:41.927Z23/30 Read block 459 len 12288
53242024-02-07T18:56:41.927Z24/30 Write block 44 len 28672 data: 3 3 4 4 5 5 5
53252024-02-07T18:56:41.927Z25/30 Write block 26 len 24576 data: 5 3 5 7 7 7
53262024-02-07T18:56:41.930Z26/30 Read block 151 len 8192
53272024-02-07T18:56:41.930Z27/30 Write block 564 len 32768 data: 4 4 4 4 4 4 5 5
53282024-02-07T18:56:41.930Z28/30 Write block 149 len 40960 data: 9 7 7 7 8 7 4 3 2 4
53292024-02-07T18:56:41.934Z29/30 Write block 330 len 20480 data: 3 4 4 5 5
53302024-02-07T18:56:41.934Z30/30 Write block 536 len 28672 data: 6 8 7 7 7 7 7
53312024-02-07T18:56:41.934Z----------------------------------------------------------------
53322024-02-07T18:56:41.934Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
53332024-02-07T18:56:41.934ZStates: Active Active Active
53342024-02-07T18:56:41.934ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
53352024-02-07T18:56:41.934Z 1 Acked 1000 Write 8 Done Done Done false
53362024-02-07T18:56:41.934Z 2 Acked 1001 Write 2 Done Done Done false
53372024-02-07T18:56:41.934Z 3 Acked 1002 Write 10 Done Done Sent false
53382024-02-07T18:56:41.934Z 4 Acked 1003 Flush 0 Done Done Sent false
53392024-02-07T18:56:41.934Z 5 Acked 1004 Flush 0 Done Done Sent false
53402024-02-07T18:56:41.934Z 6 Acked 1005 Write 9 Done Done Sent false
53412024-02-07T18:56:41.934Z 7 Acked 1006 Write 1 Done Done Sent false
53422024-02-07T18:56:41.934Z 8 Acked 1007 Write 3 Done Done Sent false
53432024-02-07T18:56:41.934Z 9 Acked 1008 Write 10 Done Done Sent false
53442024-02-07T18:56:41.934Z 10 Acked 1009 Write 5 Done Done Sent false
53452024-02-07T18:56:41.934Z 11 Acked 1010 Write 3 Done Done Sent false
53462024-02-07T18:56:41.934Z 12 Acked 1011 Read 4 Done Done Sent false
53472024-02-07T18:56:41.934Z 13 Acked 1012 Write 4 Done Done Sent false
53482024-02-07T18:56:41.934Z 14 Acked 1013 Write 2 Done Done Sent false
53492024-02-07T18:56:41.934Z 15 Acked 1014 Read 10 Done Done Sent false
53502024-02-07T18:56:41.934Z 16 Acked 1015 Write 6 Done Done Sent false
53512024-02-07T18:56:41.934Z 17 Acked 1016 Write 6 Done Done Sent false
53522024-02-07T18:56:41.935Z 18 Acked 1017 Write 9 Done Done Sent false
53532024-02-07T18:56:41.935Z 19 Acked 1018 Read 7 Done Done Sent false
53542024-02-07T18:56:41.935Z 20 Acked 1019 Read 10 Done Done Sent false
53552024-02-07T18:56:41.935Z 21 Acked 1020 Read 9 Done Done Sent false
53562024-02-07T18:56:41.935Z 22 Acked 1021 Write 3 Done Done Sent false
53572024-02-07T18:56:41.935Z 23 Acked 1022 Read 3 Done Done Sent false
53582024-02-07T18:56:41.935Z 24 Acked 1023 Write 7 Done Done Sent false
53592024-02-07T18:56:41.935Z 25 Acked 1024 Write 6 Done Done Sent false
53602024-02-07T18:56:41.935Z 26 Acked 1025 Read 2 Done Done Sent false
53612024-02-07T18:56:41.935Z 27 Acked 1026 Write 8 Done Done Sent false
53622024-02-07T18:56:41.935Z 28 Acked 1027 Write 10 Done Done Sent false
53632024-02-07T18:56:41.935Z 29 Acked 1028 Write 5 Done Done Sent false
53642024-02-07T18:56:41.935Z 30 Acked 1029 Write 7 Sent Sent Sent false
53652024-02-07T18:56:41.935Z STATES DS:0 DS:1 DS:2 TOTAL
53662024-02-07T18:56:41.935Z New 0 0 0 0
53672024-02-07T18:56:41.935Z Sent 1 1 28 30
53682024-02-07T18:56:41.935Z Done 29 29 2 60
53692024-02-07T18:56:41.935Z Skipped 0 0 0 0
53702024-02-07T18:56:41.935Z Error 0 0 0 0
53712024-02-07T18:56:41.935ZLast Flush: 1004 1004 0
53722024-02-07T18:56:41.935ZDownstairs last five completed:
53732024-02-07T18:56:41.935ZUpstairs last five completed: 30 29 28 27 26
53742024-02-07T18:56:41.935Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:41.189163363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1327}
53752024-02-07T18:56:41.935ZWrote out file "/tmp/test_repair_verify.data"
53762024-02-07T18:56:41.939Z
53772024-02-07T18:56:41.941ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53782024-02-07T18:56:41.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53792024-02-07T18:56:41.962Z 0 000-019 46 46 46 83 83 83 F F F
53802024-02-07T18:56:41.962Z 1 020-039 46 46 46 83 83 83 T T F <---
53812024-02-07T18:56:41.963Z 2 040-059 44 44 44 80 80 80 T T F <---
53822024-02-07T18:56:41.963Z 3 060-079 38 38 38 71 71 71 T T F <---
53832024-02-07T18:56:41.963Z 4 080-099 32 32 32 59 59 59 F F F
53842024-02-07T18:56:41.966Z 5 100-119 46 46 46 83 83 83 T T F <---
53852024-02-07T18:56:41.966Z 6 120-139 46 46 46 83 83 83 T T F <---
53862024-02-07T18:56:41.966Z 7 140-159 46 46 46 83 83 83 T T F <---
53872024-02-07T18:56:41.966Z 8 160-179 44 44 44 80 80 80 T T F <---
53882024-02-07T18:56:41.966Z 9 180-199 44 44 44 80 80 80 F F F
53892024-02-07T18:56:41.966Z 10 200-219 44 44 44 80 80 80 T T F <---
53902024-02-07T18:56:41.966Z 11 220-239 46 46 46 83 83 83 F F F
53912024-02-07T18:56:41.966Z 12 240-259 41 41 41 76 76 76 T T F <---
53922024-02-07T18:56:41.966Z 13 260-279 41 41 41 76 76 76 T T F <---
53932024-02-07T18:56:41.966Z 14 280-299 48 48 44 84 84 80 F F T <---
53942024-02-07T18:56:41.966Z 15 300-319 46 46 46 83 83 83 F F F
53952024-02-07T18:56:41.966Z 16 320-339 46 46 46 83 83 83 T T F <---
53962024-02-07T18:56:41.966Z 17 340-359 46 46 46 83 83 83 F F F
53972024-02-07T18:56:41.966Z 18 360-379 46 46 46 83 83 83 F F F
53982024-02-07T18:56:41.966Z 19 380-399 40 40 40 74 74 74 F F F
53992024-02-07T18:56:41.966Z 20 400-419 48 48 34 84 84 64 F F T <---
54002024-02-07T18:56:41.966Z 21 420-439 46 46 46 83 83 83 T T F <---
54012024-02-07T18:56:41.966Z 22 440-459 46 46 46 83 83 83 F F F
54022024-02-07T18:56:41.966Z 23 460-479 46 46 46 83 83 83 F F F
54032024-02-07T18:56:41.966Z 24 480-499 46 46 46 83 83 83 F F F
54042024-02-07T18:56:41.966Z 25 500-519 44 44 44 80 80 80 F F F
54052024-02-07T18:56:41.966Z 26 520-539 48 48 46 84 84 83 T T F <---
54062024-02-07T18:56:41.966Z 27 540-559 46 46 46 83 83 83 T T F <---
54072024-02-07T18:56:41.967Z 28 560-579 46 46 46 83 83 83 T T F <---
54082024-02-07T18:56:41.967Z 29 580-599 45 45 45 82 82 82 F F F
54092024-02-07T18:56:41.967ZMax gen: 48, Max flush: 84
54102024-02-07T18:56:41.967ZError: Difference in extent metadata found!
54112024-02-07T18:56:41.967ZA Difference in extent metadata is expected here
54122024-02-07T18:56:41.967ZOn loop 24
54132024-02-07T18:56:41.967Z
54142024-02-07T18:56:41.970ZVerifying data now
54152024-02-07T18:56:47.808ZNo change after verify
54162024-02-07T18:56:47.808ZLoop: 24 Downstairs dump after verify (and repair):
54172024-02-07T18:56:47.822ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54182024-02-07T18:56:47.822Z 0 000-019 46 46 46 83 83 83 F F F
54192024-02-07T18:56:47.822Z 1 020-039 49 49 49 85 85 85 F F F
54202024-02-07T18:56:47.822Z 2 040-059 49 49 49 85 85 85 F F F
54212024-02-07T18:56:47.823Z 3 060-079 49 49 49 85 85 85 F F F
54222024-02-07T18:56:47.823Z 4 080-099 32 32 32 59 59 59 F F F
54232024-02-07T18:56:47.823Z 5 100-119 49 49 49 85 85 85 F F F
54242024-02-07T18:56:47.823Z 6 120-139 49 49 49 85 85 85 F F F
54252024-02-07T18:56:47.823Z 7 140-159 49 49 49 85 85 85 F F F
54262024-02-07T18:56:47.823Z 8 160-179 49 49 49 85 85 85 F F F
54272024-02-07T18:56:47.823Z 9 180-199 44 44 44 80 80 80 F F F
54282024-02-07T18:56:47.823Z 10 200-219 49 49 49 85 85 85 F F F
54292024-02-07T18:56:47.823Z 11 220-239 46 46 46 83 83 83 F F F
54302024-02-07T18:56:47.823Z 12 240-259 49 49 49 85 85 85 F F F
54312024-02-07T18:56:47.823Z 13 260-279 49 49 49 85 85 85 F F F
54322024-02-07T18:56:47.823Z 14 280-299 48 48 48 84 84 84 F F F
54332024-02-07T18:56:47.824Z 15 300-319 46 46 46 83 83 83 F F F
54342024-02-07T18:56:47.824Z 16 320-339 49 49 49 85 85 85 F F F
54352024-02-07T18:56:47.824Z 17 340-359 46 46 46 83 83 83 F F F
54362024-02-07T18:56:47.827Z 18 360-379 46 46 46 83 83 83 F F F
54372024-02-07T18:56:47.828Z 19 380-399 40 40 40 74 74 74 F F F
54382024-02-07T18:56:47.828Z 20 400-419 48 48 48 84 84 84 F F F
54392024-02-07T18:56:47.828Z 21 420-439 49 49 49 85 85 85 F F F
54402024-02-07T18:56:47.828Z 22 440-459 46 46 46 83 83 83 F F F
54412024-02-07T18:56:47.828Z 23 460-479 46 46 46 83 83 83 F F F
54422024-02-07T18:56:47.828Z 24 480-499 46 46 46 83 83 83 F F F
54432024-02-07T18:56:47.828Z 25 500-519 44 44 44 80 80 80 F F F
54442024-02-07T18:56:47.828Z 26 520-539 49 49 49 85 85 85 F F F
54452024-02-07T18:56:47.828Z 27 540-559 49 49 49 85 85 85 F F F
54462024-02-07T18:56:47.828Z 28 560-579 49 49 49 85 85 85 F F F
54472024-02-07T18:56:47.828Z 29 580-599 45 45 45 82 82 82 F F F
54482024-02-07T18:56:47.828ZMax gen: 49, Max flush: 85
54492024-02-07T18:56:47.829Z
54502024-02-07T18:56:47.829ZBegin loop 25
54512024-02-07T18:56:47.829ZDownstairs to restart: 1
54522024-02-07T18:56:47.844Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099149524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335}
54532024-02-07T18:56:47.844Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099393851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335}
54542024-02-07T18:56:47.844Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099412688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335}
54552024-02-07T18:56:47.844Z{"msg":"Crucible ec2ac59d-7076-43f9-9f2b-38a68cfaf6e6 has session id: 2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099426637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54562024-02-07T18:56:47.849Z{"msg":"Upstairs opts: Upstairs UUID: ec2ac59d-7076-43f9-9f2b-38a68cfaf6e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"timeCrucible runtime is spawned
54572024-02-07T18:56:47.850Z":"2024-02-07T18:56:47.099471571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54582024-02-07T18:56:47.850Z{"msg":"Crucible stats registered with UUID: ec2ac59d-7076-43f9-9f2b-38a68cfaf6e6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099508255Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54592024-02-07T18:56:47.850Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099555259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335}
54602024-02-07T18:56:47.850Z{"msg":"ec2ac59d-7076-43f9-9f2b-38a68cfaf6e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099677613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54612024-02-07T18:56:47.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099714678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54622024-02-07T18:56:47.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099733145Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54632024-02-07T18:56:47.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099762931Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54642024-02-07T18:56:47.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099786338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54652024-02-07T18:56:47.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099811765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54662024-02-07T18:56:47.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099834662Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54672024-02-07T18:56:47.850Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099911331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54682024-02-07T18:56:47.850Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099956295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54692024-02-07T18:56:47.851Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.099987391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54702024-02-07T18:56:47.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100014697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54712024-02-07T18:56:47.851Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:47.100044053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54722024-02-07T18:56:47.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100081998Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54732024-02-07T18:56:47.851Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:47.100109744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54742024-02-07T18:56:47.851Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:47.10013868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54752024-02-07T18:56:47.851Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100152898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54762024-02-07T18:56:47.851Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100179725Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54772024-02-07T18:56:47.851Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100270612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54782024-02-07T18:56:47.851Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100603807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54792024-02-07T18:56:47.851Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.10065985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54802024-02-07T18:56:47.851Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100686776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54812024-02-07T18:56:47.852Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.100720332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54822024-02-07T18:56:47.852Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101162402Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54832024-02-07T18:56:47.852Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101282806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54842024-02-07T18:56:47.852Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101627599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54852024-02-07T18:56:47.852Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101676162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54862024-02-07T18:56:47.852Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101705398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54872024-02-07T18:56:47.852Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101740044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54882024-02-07T18:56:47.852Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.10176931Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54892024-02-07T18:56:47.852Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:47.101806735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54902024-02-07T18:56:48.852Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.100816228Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54912024-02-07T18:56:48.852Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.10094465Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54922024-02-07T18:56:49.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.102027754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54932024-02-07T18:56:49.266Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.102061939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54942024-02-07T18:56:49.266Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.102587988Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54952024-02-07T18:56:49.266Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103212774Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54962024-02-07T18:56:49.266Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103260937Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54972024-02-07T18:56:49.266Z{"msg":"[0]R flush_numbers[0..12]: [83, 85, 85, 85, 59, 85, 85, 85, 85, 80, 85, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103283504Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
54982024-02-07T18:56:49.266Z{"msg":"[0]R generation[0..12]: [46, 49, 49, 49, 32, 49, 49, 49, 49, 44, 49, 46]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
54992024-02-07T18:56:49.266Z,"time":"2024-02-07T18:56:48.103334727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55002024-02-07T18:56:49.266Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103378371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55012024-02-07T18:56:49.266Z{"msg":"[1]R flush_numbers[0..12]: [83, 85, 85, 85, 59, 85, 85, 85, 85, 80, 85, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103408427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55022024-02-07T18:56:49.266Z{"msg":"[1]R generation[0..12]: [46, 49, 49, 49, 32, 49, 49, 49, 49, 44, 49, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103449331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55032024-02-07T18:56:49.266Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103487686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55042024-02-07T18:56:49.267Z{"msg":"[2]R flush_numbers[0..12]: [83, 85, 85, 85, 59, 85, 85, 85, 85, 80, 85, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103525951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55052024-02-07T18:56:49.267Z{"msg":"[2]R generation[0..12]: [46, 49, 49, 49, 32, 49, 49, 49, 49, 44, 49, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103563496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55062024-02-07T18:56:49.267Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103602341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55072024-02-07T18:56:49.267Z{"msg":"Max found gen is 50","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
55082024-02-07T18:56:49.267Zcrucible","level":30,"time":"2024-02-07T18:56:48.103641056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55092024-02-07T18:56:49.267Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103661823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55102024-02-07T18:56:49.267Z{"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.10368441Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55112024-02-07T18:56:49.267Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103700237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55122024-02-07T18:56:49.267Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103721245Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55132024-02-07T18:56:49.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103733883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55142024-02-07T18:56:49.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.10375405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55152024-02-07T18:56:49.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103767868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55162024-02-07T18:56:49.267Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103788816Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55172024-02-07T18:56:49.268Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103802164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55182024-02-07T18:56:49.268Z{"msg":"ec2ac59d-7076-43f9-9f2b-38a68cfaf6e6 is now active with session: 2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103822681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55192024-02-07T18:56:49.268Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103835039Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335,"session_id":"2d3beef6-57a9-4d9d-9039-f3d8b9a2e2ce"}
55202024-02-07T18:56:49.268Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:48.103853927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335}
55212024-02-07T18:56:49.268ZLoading write count information from file "/tmp/test_repair_verify.data"
55222024-02-07T18:56:49.268ZRun Repair workload
55232024-02-07T18:56:49.268Z01/30 Write block 27 len 40960 data: 4 6 8 8 8 6 6 6 6 6
55242024-02-07T18:56:49.268Z02/30 Write block 547 len 16384 data: 3 4 5 5
55252024-02-07T18:56:49.268Z03/30 Flush
55262024-02-07T18:56:49.268Z04/30 Write block 55 len 16384 data: 8 7 7 6
55272024-02-07T18:56:49.268Z05/30 Write block 580 len 16384 data: 4 6 6 5
55282024-02-07T18:56:49.268Z06/30 Read block 585 len 24576
55292024-02-07T18:56:49.269Z07/30 Write block 218 len 4096 data: 3
55302024-02-07T18:56:49.269Z08/30 Write block 140 len 40960 data: 2 5 5 5 5 5 7 7 7 10
55312024-02-07T18:56:49.282Z09/30 Read block 588 len 40960
55322024-02-07T18:56:49.283Z10/30 Read block 514 len 8192
55332024-02-07T18:56:49.283Z11/30 Read block 280 len 24576
55342024-02-07T18:56:49.283Z12/30 Write block 204 len 16384 data: 6 8 7 6
55352024-02-07T18:56:49.283Z13/30 Read block 78 len 28672
55362024-02-07T18:56:49.283Z14/30 Read block 141 len 36864
55372024-02-07T18:56:49.283Z15/30 Write block 470 len 8192 data: 3 3
55382024-02-07T18:56:49.283Z16/30 Read block 206 len 32768
55392024-02-07T18:56:49.283Z17/30 Read block 467 len 36864
55402024-02-07T18:56:49.283Z18/30 Read block 85 len 36864
55412024-02-07T18:56:49.283Z19/30 Read block 17 len 20480
55422024-02-07T18:56:49.284Z20/30 Write block 400 len 12288 data: 4 3 4
55432024-02-07T18:56:49.284Z21/30 Write block 159 len 40960 data: 4 4 5 7 8 8 8 7 7 8
55442024-02-07T18:56:49.284Z22/30 Read block 172 len 8192
55452024-02-07T18:56:49.284Z23/30 Write block 66 len 28672 data: 7 7 7 8 7 8 6
55462024-02-07T18:56:49.284Z24/30 Write block 56 len 24576 data: 8 8 7 5 4 4
55472024-02-07T18:56:49.284Z25/30 Read block 293 len 20480
55482024-02-07T18:56:49.284Z26/30 Write block 59 len 32768 data: 6 5 5 4 4 5 5 8
55492024-02-07T18:56:49.284Z27/30 Read block 137 len 12288
55502024-02-07T18:56:49.284Z28/30 Write block 588 len 32768 data: 5 5 5 6 6 5 5 4
55512024-02-07T18:56:49.284Z29/30 Write block 529 len 32768 data: 4 4 4 5 5 5 6 7
55522024-02-07T18:56:49.284Z30/30 Write block 423 len 16384 data: 3 4 6 6
55532024-02-07T18:56:49.284Z----------------------------------------------------------------
55542024-02-07T18:56:49.284Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
55552024-02-07T18:56:49.284ZStates: Active Active Active
55562024-02-07T18:56:49.285ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55572024-02-07T18:56:49.285Z 1 Acked 1000 Write 10 Done Sent Done false
55582024-02-07T18:56:49.285Z 2 Acked 1001 Write 4 Done Sent Done false
55592024-02-07T18:56:49.285Z 3 Acked 1002 Flush 0 Done Sent Done false
55602024-02-07T18:56:49.285Z 4 Acked 1003 Write 4 Done Sent Done false
55612024-02-07T18:56:49.285Z 5 Acked 1004 Write 4 Done Sent Done false
55622024-02-07T18:56:49.285Z 6 Acked 1005 Read 6 Done Sent Done false
55632024-02-07T18:56:49.285Z 7 Acked 1006 Write 1 Done Sent Done false
55642024-02-07T18:56:49.285Z 8 Acked 1007 Write 10 Done Sent Done false
55652024-02-07T18:56:49.286Z 9 Acked 1008 Read 10 Done Sent Done false
55662024-02-07T18:56:49.286Z 10 Acked 1009 Read 2 Done Sent Done false
55672024-02-07T18:56:49.286Z 11 Acked 1010 Read 6 Done Sent Done false
55682024-02-07T18:56:49.286Z 12 Acked 1011 Write 4 Done Sent Done false
55692024-02-07T18:56:49.286Z 13 Acked 1012 Read 7 Done Sent Done false
55702024-02-07T18:56:49.286Z 14 Acked 1013 Read 9 Done Sent Done false
55712024-02-07T18:56:49.286Z 15 Acked 1014 Write 2 Done Sent Done false
55722024-02-07T18:56:49.286Z 16 Acked 1015 Read 8 Done Sent Done false
55732024-02-07T18:56:49.286Z 17 Acked 1016 Read 9 Done Sent Done false
55742024-02-07T18:56:49.286Z 18 Acked 1017 Read 9 Done Sent Done false
55752024-02-07T18:56:49.286Z 19 Acked 1018 Read 5 Done Sent Done false
55762024-02-07T18:56:49.286Z 20 Acked 1019 Write 3 Done Sent Done false
55772024-02-07T18:56:49.286Z 21 Acked 1020 Write 10 Done Sent Done false
55782024-02-07T18:56:49.286Z 22 Acked 1021 Read 2 Done Sent Done false
55792024-02-07T18:56:49.286Z 23 Acked 1022 Write 7 Done Sent Done false
55802024-02-07T18:56:49.286Z 24 Acked 1023 Write 6 Done Sent Done false
55812024-02-07T18:56:49.287Z 25 Acked 1024 Read 5 Done Sent Done false
55822024-02-07T18:56:49.287Z 26 Acked 1025 Write 8 Done Sent Done false
55832024-02-07T18:56:49.287Z 27 Acked 1026 Read 3 Done Sent Done false
55842024-02-07T18:56:49.287Z 28 Acked 1027 Write 8 Done Sent Done false
55852024-02-07T18:56:49.287Z 29 Acked 1028 Write 8 Sent Sent Sent false
55862024-02-07T18:56:49.287Z 30 Acked 1029 Write 4 Sent Sent Sent false
55872024-02-07T18:56:49.287Z STATES DS:0 DS:1 DS:2 TOTAL
55882024-02-07T18:56:49.287Z New 0 0 0 0
55892024-02-07T18:56:49.287Z Sent 2 30 2 34
55902024-02-07T18:56:49.287Z Done 28 0 28 56
55912024-02-07T18:56:49.287Z Skipped 0 0 0 0
55922024-02-07T18:56:49.287Z Error 0 0 0 0
55932024-02-07T18:56:49.287ZLast Flush: 1002 0 1002
55942024-02-07T18:56:49.288ZDownstairs last five completed:
55952024-02-07T18:56:49.288ZUpstairs last five completed: 30 29 28 27 26
55962024-02-07T18:56:49.288Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:48.133843671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1335}
55972024-02-07T18:56:49.288ZWrote out file "/tmp/test_repair_verify.data"
55982024-02-07T18:56:49.288Z
55992024-02-07T18:56:49.288ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
56002024-02-07T18:56:49.288ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56012024-02-07T18:56:49.288Z 0 000-019 46 46 46 83 83 83 F F F
56022024-02-07T18:56:49.288Z 1 020-039 50 49 50 86 85 86 F F F <---
56032024-02-07T18:56:49.288Z 2 040-059 49 49 49 85 85 85 T F T <---
56042024-02-07T18:56:49.288Z 3 060-079 49 49 49 85 85 85 T F T <---
56052024-02-07T18:56:49.288Z 4 080-099 32 32 32 59 59 59 F F F
56062024-02-07T18:56:49.289Z 5 100-119 49 49 49 85 85 85 F F F
56072024-02-07T18:56:49.289Z 6 120-139 49 49 49 85 85 85 F F F
56082024-02-07T18:56:49.289Z 7 140-159 49 49 49 85 85 85 T F T <---
56092024-02-07T18:56:49.289Z 8 160-179 49 49 49 85 85 85 T F T <---
56102024-02-07T18:56:49.289Z 9 180-199 44 44 44 80 80 80 F F F
56112024-02-07T18:56:49.289Z 10 200-219 49 49 49 85 85 85 T F T <---
56122024-02-07T18:56:49.289Z 11 220-239 46 46 46 83 83 83 F F F
56132024-02-07T18:56:49.289Z 12 240-259 49 49 49 85 85 85 F F F
56142024-02-07T18:56:49.289Z 13 260-279 49 49 49 85 85 85 F F F
56152024-02-07T18:56:49.289Z 14 280-299 48 48 48 84 84 84 F F F
56162024-02-07T18:56:49.289Z 15 300-319 46 46 46 83 83 83 F F F
56172024-02-07T18:56:49.289Z 16 320-339 49 49 49 85 85 85 F F F
56182024-02-07T18:56:49.289Z 17 340-359 46 46 46 83 83 83 F F F
56192024-02-07T18:56:49.289Z 18 360-379 46 46 46 83 83 83 F F F
56202024-02-07T18:56:49.290Z 19 380-399 40 40 40 74 74 74 F F F
56212024-02-07T18:56:49.290Z 20 400-419 48 48 48 84 84 84 T F T <---
56222024-02-07T18:56:49.290Z 21 420-439 49 49 49 85 85 85 T F T <---
56232024-02-07T18:56:49.290Z 22 440-459 46 46 46 83 83 83 F F F
56242024-02-07T18:56:49.290Z 23 460-479 46 46 46 83 83 83 T F T <---
56252024-02-07T18:56:49.290Z 24 480-499 46 46 46 83 83 83 F F F
56262024-02-07T18:56:49.290Z 25 500-519 44 44 44 80 80 80 F F F
56272024-02-07T18:56:49.290Z 26 520-539 49 49 49 85 85 85 T F T <---
56282024-02-07T18:56:49.290Z 27 540-559 50 49 50 86 85 86 F F F <---
56292024-02-07T18:56:49.290Z 28 560-579 49 49 49 85 85 85 F F F
56302024-02-07T18:56:49.290Z 29 580-599 45 45 45 82 82 82 T F T <---
56312024-02-07T18:56:49.294ZMax gen: 50, Max flush: 86
56322024-02-07T18:56:49.295ZError: Difference in extent metadata found!
56332024-02-07T18:56:49.295ZA Difference in extent metadata is expected here
56342024-02-07T18:56:49.295ZOn loop 25
56352024-02-07T18:56:49.295Z
56362024-02-07T18:56:49.295ZVerifying data now
56372024-02-07T18:56:53.525ZNo change after verify
56382024-02-07T18:56:53.525ZLoop: 25 Downstairs dump after verify (and repair):
56392024-02-07T18:56:53.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56402024-02-07T18:56:53.545Z 0 000-019 46 46 46 83 83 83 F F F
56412024-02-07T18:56:53.545Z 1 020-039 50 50 50 86 86 86 F F F
56422024-02-07T18:56:53.545Z 2 040-059 51 51 51 87 87 87 F F F
56432024-02-07T18:56:53.545Z 3 060-079 51 51 51 87 87 87 F F F
56442024-02-07T18:56:53.545Z 4 080-099 32 32 32 59 59 59 F F F
56452024-02-07T18:56:53.545Z 5 100-119 49 49 49 85 85 85 F F F
56462024-02-07T18:56:53.545Z 6 120-139 49 49 49 85 85 85 F F F
56472024-02-07T18:56:53.545Z 7 140-159 51 51 51 87 87 87 F F F
56482024-02-07T18:56:53.546Z 8 160-179 51 51 51 87 87 87 F F F
56492024-02-07T18:56:53.546Z 9 180-199 44 44 44 80 80 80 F F F
56502024-02-07T18:56:53.546Z 10 200-219 51 51 51 87 87 87 F F F
56512024-02-07T18:56:53.546Z 11 220-239 46 46 46 83 83 83 F F F
56522024-02-07T18:56:53.546Z 12 240-259 49 49 49 85 85 85 F F F
56532024-02-07T18:56:53.553Z 13 260-279 49 49 49 85 85 85 F F F
56542024-02-07T18:56:53.553Z 14 280-299 48 48 48 84 84 84 F F F
56552024-02-07T18:56:53.553Z 15 300-319 46 46 46 83 83 83 F F F
56562024-02-07T18:56:53.553Z 16 320-339 49 49 49 85 85 85 F F F
56572024-02-07T18:56:53.553Z 17 340-359 46 46 46 83 83 83 F F F
56582024-02-07T18:56:53.553Z 18 360-379 46 46 46 83 83 83 F F F
56592024-02-07T18:56:53.553Z 19 380-399 40 40 40 74 74 74 F F F
56602024-02-07T18:56:53.553Z 20 400-419 51 51 51 87 87 87 F F F
56612024-02-07T18:56:53.554Z 21 420-439 51 51 51 87 87 87 F F F
56622024-02-07T18:56:53.554Z 22 440-459 46 46 46 83 83 83 F F F
56632024-02-07T18:56:53.554Z 23 460-479 51 51 51 87 87 87 F F F
56642024-02-07T18:56:53.554Z 24 480-499 46 46 46 83 83 83 F F F
56652024-02-07T18:56:53.554Z 25 500-519 44 44 44 80 80 80 F F F
56662024-02-07T18:56:53.554Z 26 520-539 51 51 51 87 87 87 F F F
56672024-02-07T18:56:53.554Z 27 540-559 50 50 50 86 86 86 F F F
56682024-02-07T18:56:53.554Z 28 560-579 49 49 49 85 85 85 F F F
56692024-02-07T18:56:53.554Z 29 580-599 51 51 51 87 87 87 F F F
56702024-02-07T18:56:53.554ZMax gen: 51, Max flush: 87
56712024-02-07T18:56:53.554Z
56722024-02-07T18:56:53.554ZBegin loop 26
56732024-02-07T18:56:53.554ZDownstairs to restart: 0
56742024-02-07T18:56:53.562Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817052324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343}
56752024-02-07T18:56:53.563Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
56762024-02-07T18:56:53.563Z,"time":"2024-02-07T18:56:52.81737317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343}
56772024-02-07T18:56:53.568Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817407936Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343}
56782024-02-07T18:56:53.568Z{"msg":"Crucible 10ecf0ab-d8e7-4b77-bf96-be0f0c0310b2 has session id: 47e4fb9c-b972-478a-9090-7d8050589ffc","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817462838Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56792024-02-07T18:56:53.568Z{"msg":"Upstairs opts: Upstairs UUID: 10ecf0ab-d8e7-4b77-bf96-be0f0c0310b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817513241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56802024-02-07T18:56:53.568Z{"msg":"Crucible stats registered with UUID: 10ecf0ab-d8e7-4b77-bf96-be0f0c0310b2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817533679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56812024-02-07T18:56:53.568Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817560485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343}
56822024-02-07T18:56:53.568Z{"msg":"10ecf0ab-d8e7-4b77-bf96-be0f0c0310b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817684398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56832024-02-07T18:56:53.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817716864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56842024-02-07T18:56:53.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817735881Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56852024-02-07T18:56:53.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817753669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56862024-02-07T18:56:53.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817795903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56872024-02-07T18:56:53.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.81781723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56882024-02-07T18:56:53.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817834518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56892024-02-07T18:56:53.569Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817851216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56902024-02-07T18:56:53.569Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817867953Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56912024-02-07T18:56:53.569Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817882152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56922024-02-07T18:56:53.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.817994556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56932024-02-07T18:56:53.569Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:52.818081174Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56942024-02-07T18:56:53.569Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:52.818154075Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56952024-02-07T18:56:53.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818203738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56962024-02-07T18:56:53.569Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:52.818238803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56972024-02-07T18:56:53.569Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818267619Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56982024-02-07T18:56:53.569Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818284447Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
56992024-02-07T18:56:53.569Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818324652Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57002024-02-07T18:56:53.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818738716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57012024-02-07T18:56:53.569Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818765692Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57022024-02-07T18:56:53.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.81878042Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57032024-02-07T18:56:53.569Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.818794068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57042024-02-07T18:56:53.569Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.81929884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57052024-02-07T18:56:53.569Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.819411015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57062024-02-07T18:56:53.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.819834317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57072024-02-07T18:56:53.569Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.819883451Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57082024-02-07T18:56:53.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.819913847Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57092024-02-07T18:56:53.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.819954801Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57102024-02-07T18:56:53.569Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.820006274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57112024-02-07T18:56:53.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:52.820032531Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57122024-02-07T18:56:54.565Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.818862619Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57132024-02-07T18:56:54.566Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.818950517Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57142024-02-07T18:56:54.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.819802991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57152024-02-07T18:56:54.572Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.819841086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57162024-02-07T18:56:54.572Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.820363436Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57172024-02-07T18:56:54.572Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.820946007Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57182024-02-07T18:56:54.572Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.820985931Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57192024-02-07T18:56:54.572Z{"msg":"[0]R flush_numbers[0..12]: [83, 86, 87, 87, 59, 85, 85, 87, 87, 80, 87, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821006319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57202024-02-07T18:56:54.572Z{"msg":"[0]R generation[0..12]: [46, 50, 51, 51, 32, 49, 49, 51, 51, 44, 51, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821047713Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57212024-02-07T18:56:54.572Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821088968Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57222024-02-07T18:56:54.572Z{"msg":"[1]R flush_numbers[0..12]: [83, 86, 87, 87, 59, 85, 85, 87, 87, 80, 87, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821130662Z"Wait for a query_work_queue command to finish before sending IO
57232024-02-07T18:56:54.572Z,"hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57242024-02-07T18:56:54.572Z{"msg":"[1]R generation[0..12]: [46, 50, 51, 51, 32, 49, 49, 51, 51, 44, 51, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821174446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57252024-02-07T18:56:54.573Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821215691Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57262024-02-07T18:56:54.573Z{"msg":"[2]R flush_numbers[0..12]: [83, 86, 87, 87, 59, 85, 85, 87, 87, 80, 87, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821236708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57272024-02-07T18:56:54.573Z{"msg":"[2]R generation[0..12]: [46, 50, 51, 51, 32, 49, 49, 51, 51, 44, 51, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821279142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57282024-02-07T18:56:54.573Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821324886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57292024-02-07T18:56:54.573Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.82136905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57302024-02-07T18:56:54.573Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821422832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57312024-02-07T18:56:54.573Z{"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821466986Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57322024-02-07T18:56:54.573Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821507461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57332024-02-07T18:56:54.573Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821526158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57342024-02-07T18:56:54.573Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821561564Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57352024-02-07T18:56:54.573Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.82158724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57362024-02-07T18:56:54.573Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821625845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57372024-02-07T18:56:54.573Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821645332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57382024-02-07T18:56:54.573Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821667659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57392024-02-07T18:56:54.573ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
57402024-02-07T18:56:54.573Z{"msg":"10ecf0ab-d8e7-4b77-bf96-be0f0c0310b2 is now active with session: 47e4fb9c-b972-478a-9090-7d8050589ffc","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821712103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57412024-02-07T18:56:54.573Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821732661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343,"session_id":"47e4fb9c-b972-478a-9090-7d8050589ffc"}
57422024-02-07T18:56:54.573Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:53.821756927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343}
57432024-02-07T18:56:54.573ZLoading write count information from file "/tmp/test_repair_verify.data"
57442024-02-07T18:56:54.573ZRun Repair workload
57452024-02-07T18:56:54.573Z01/30 Write block 524 len 12288 data: 3 2 3
57462024-02-07T18:56:54.573Z02/30 Write block 441 len 36864 data: 6 8 8 7 5 5 3 3 3
57472024-02-07T18:56:54.573Z03/30 Read block 523 len 32768
57482024-02-07T18:56:54.577Z04/30 Read block 126 len 12288
57492024-02-07T18:56:54.577Z05/30 Write block 40 len 24576 data: 4 3 4 4 4 4
57502024-02-07T18:56:54.577Z06/30 Write block 542 len 32768 data: 8 4 4 2 2 4 5 6
57512024-02-07T18:56:54.577Z07/30 Read block 424 len 16384
57522024-02-07T18:56:54.577Z08/30 Write block 229 len 8192 data: 4 4
57532024-02-07T18:56:54.577Z09/30 Read block 578 len 20480
57542024-02-07T18:56:54.577Z10/30 Write block 308 len 4096 data: 5
57552024-02-07T18:56:54.577Z11/30 Write block 204 len 20480 data: 7 9 8 7 8
57562024-02-07T18:56:54.581Z12/30 Write block 553 len 4096 data: 5
57572024-02-07T18:56:54.581Z13/30 Read block 297 len 36864
57582024-02-07T18:56:54.581Z14/30 Read block 76 len 12288
57592024-02-07T18:56:54.581Z15/30 Flush
57602024-02-07T18:56:54.585Z16/30 Write block 475 len 8192 data: 3 4
57612024-02-07T18:56:54.585Z17/30 Read block 158 len 12288
57622024-02-07T18:56:54.585Z18/30 Write block 92 len 28672 data: 6 5 7 6 5 5 4
57632024-02-07T18:56:54.590Z19/30 Write block 542 len 8192 data: 9 5
57642024-02-07T18:56:54.590Z20/30 Read block 355 len 20480
57652024-02-07T18:56:54.590Z21/30 Read block 25 len 40960
57662024-02-07T18:56:54.590Z22/30 Write block 478 len 8192 data: 4 3
57672024-02-07T18:56:54.590Z23/30 Write block 28 len 28672 data: 7 9 9 9 7 7 7
57682024-02-07T18:56:54.590Z24/30 Read block 520 len 32768
57692024-02-07T18:56:54.594Z25/30 Write block 545 len 36864 data: 3 3 5 6 7 6 5 5 6
57702024-02-07T18:56:54.594Z26/30 Read block 485 len 40960
57712024-02-07T18:56:54.594Z27/30 Write block 114 len 16384 data: 2 4 4 5
57722024-02-07T18:56:54.594Z28/30 Read block 12 len 12288
57732024-02-07T18:56:54.599Z29/30 Read block 115 len 40960
57742024-02-07T18:56:54.600Z30/30 Write block 387 len 32768 data: 5 6 7 5 5 5 7 6
57752024-02-07T18:56:54.600Z----------------------------------------------------------------
57762024-02-07T18:56:54.600Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57772024-02-07T18:56:54.600ZStates: Active Active Active
57782024-02-07T18:56:54.600ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57792024-02-07T18:56:54.600Z 1 Acked 1000 Write 3 Done Done Done false
57802024-02-07T18:56:54.600Z 2 Acked 1001 Write 9 Done Done Done false
57812024-02-07T18:56:54.600Z 3 Acked 1002 Read 8 Done Done Done false
57822024-02-07T18:56:54.600Z 4 Acked 1003 Read 3 Sent Done Done false
57832024-02-07T18:56:54.600Z 5 Acked 1004 Write 6 Sent Done Done false
57842024-02-07T18:56:54.600Z 6 Acked 1005 Write 8 Sent Done Done false
57852024-02-07T18:56:54.600Z 7 Acked 1006 Read 4 Sent Done Done false
57862024-02-07T18:56:54.600Z 8 Acked 1007 Write 2 Sent Done Done false
57872024-02-07T18:56:54.600Z 9 Acked 1008 Read 5 Sent Done Done false
57882024-02-07T18:56:54.600Z 10 Acked 1009 Write 1 Sent Done Done false
57892024-02-07T18:56:54.600Z 11 Acked 1010 Write 5 Sent Done Done false
57902024-02-07T18:56:54.600Z 12 Acked 1011 Write 1 Sent Done Done false
57912024-02-07T18:56:54.600Z 13 Acked 1012 Read 9 Sent Done Done false
57922024-02-07T18:56:54.600Z 14 Acked 1013 Read 3 Sent Done Done false
57932024-02-07T18:56:54.600Z 15 Acked 1014 Flush 0 Sent Done Done false
57942024-02-07T18:56:54.600Z 16 Acked 1015 Write 2 Sent Done Done false
57952024-02-07T18:56:54.600Z 17 Acked 1016 Read 3 Sent Done Done false
57962024-02-07T18:56:54.600Z 18 Acked 1017 Write 7 Sent Done Done false
57972024-02-07T18:56:54.601Z 19 Acked 1018 Write 2 Sent Done Done false
57982024-02-07T18:56:54.601Z 20 Acked 1019 Read 5 Sent Done Done false
57992024-02-07T18:56:54.601Z 21 Acked 1020 Read 10 Sent Done Done false
58002024-02-07T18:56:54.601Z 22 Acked 1021 Write 2 Sent Done Done false
58012024-02-07T18:56:54.601Z 23 Acked 1022 Write 7 Sent Done Done false
58022024-02-07T18:56:54.601Z 24 Acked 1023 Read 8 Sent Done Done false
58032024-02-07T18:56:54.601Z 25 Acked 1024 Write 9 Sent Done Done false
58042024-02-07T18:56:54.601Z 26 Acked 1025 Read 10 Sent Done Done false
58052024-02-07T18:56:54.601Z 27 Acked 1026 Write 4 Sent Done Done false
58062024-02-07T18:56:54.601Z 28 Acked 1027 Read 3 Sent Done Done false
58072024-02-07T18:56:54.601Z 29 Acked 1028 Read 10 Sent Done Done false
58082024-02-07T18:56:54.601Z 30 Acked 1029 Write 8 Sent Sent Sent false
58092024-02-07T18:56:54.601Z STATES DS:0 DS:1 DS:2 TOTAL
58102024-02-07T18:56:54.601Z New 0 0 0 0
58112024-02-07T18:56:54.601Z Sent 27 1 1 29
58122024-02-07T18:56:54.601Z Done 3 29 29 61
58132024-02-07T18:56:54.601Z Skipped 0 0 0 0
58142024-02-07T18:56:54.601Z Error 0 0 0 0
58152024-02-07T18:56:54.601ZLast Flush: 0 1014 1014
58162024-02-07T18:56:54.601ZDownstairs last five completed:
58172024-02-07T18:56:54.601ZUpstairs last five completed: 30 29 28 27 26
58182024-02-07T18:56:54.601Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:53.851095469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1343}
58192024-02-07T18:56:54.601ZWrote out file "/tmp/test_repair_verify.data"
58202024-02-07T18:56:54.605Z
58212024-02-07T18:56:54.610ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
58222024-02-07T18:56:55.034ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58232024-02-07T18:56:55.034Z 0 000-019 46 46 46 83 83 83 F F F
58242024-02-07T18:56:55.034Z 1 020-039 50 50 50 86 86 86 F T T <---
58252024-02-07T18:56:55.034Z 2 040-059 51 52 52 87 88 88 F F F <---
58262024-02-07T18:56:55.034Z 3 060-079 51 51 51 87 87 87 F F F
58272024-02-07T18:56:55.034Z 4 080-099 32 32 32 59 59 59 F T T <---
58282024-02-07T18:56:55.034Z 5 100-119 49 49 49 85 85 85 F T T <---
58292024-02-07T18:56:55.035Z 6 120-139 49 49 49 85 85 85 F F F
58302024-02-07T18:56:55.035Z 7 140-159 51 51 51 87 87 87 F F F
58312024-02-07T18:56:55.035Z 8 160-179 51 51 51 87 87 87 F F F
58322024-02-07T18:56:55.035Z 9 180-199 44 44 44 80 80 80 F F F
58332024-02-07T18:56:55.038Z 10 200-219 51 52 52 87 88 88 F F F <---
58342024-02-07T18:56:55.038Z 11 220-239 46 52 52 83 88 88 F F F <---
58352024-02-07T18:56:55.039Z 12 240-259 49 49 49 85 85 85 F F F
58362024-02-07T18:56:55.039Z 13 260-279 49 49 49 85 85 85 F F F
58372024-02-07T18:56:55.039Z 14 280-299 48 48 48 84 84 84 F F F
58382024-02-07T18:56:55.039Z 15 300-319 46 52 52 83 88 88 F F F <---
58392024-02-07T18:56:55.039Z 16 320-339 49 49 49 85 85 85 F F F
58402024-02-07T18:56:55.039Z 17 340-359 46 46 46 83 83 83 F F F
58412024-02-07T18:56:55.039Z 18 360-379 46 46 46 83 83 83 F F F
58422024-02-07T18:56:55.039Z 19 380-399 40 40 40 74 74 74 F T T <---
58432024-02-07T18:56:55.039Z 20 400-419 51 51 51 87 87 87 F F F
58442024-02-07T18:56:55.039Z 21 420-439 51 51 51 87 87 87 F F F
58452024-02-07T18:56:55.039Z 22 440-459 46 52 52 83 88 88 T F F <---
58462024-02-07T18:56:55.039Z 23 460-479 51 51 51 87 87 87 F T T <---
58472024-02-07T18:56:55.039Z 24 480-499 46 46 46 83 83 83 F F F
58482024-02-07T18:56:55.039Z 25 500-519 44 44 44 80 80 80 F F F
58492024-02-07T18:56:55.039Z 26 520-539 51 52 52 87 88 88 T F F <---
58502024-02-07T18:56:55.039Z 27 540-559 50 52 52 86 88 88 F T T <---
58512024-02-07T18:56:55.039Z 28 560-579 49 49 49 85 85 85 F F F
58522024-02-07T18:56:55.039Z 29 580-599 51 51 51 87 87 87 F F F
58532024-02-07T18:56:55.039ZMax gen: 52, Max flush: 88
58542024-02-07T18:56:55.039ZError: Difference in extent metadata found!
58552024-02-07T18:56:55.039ZA Difference in extent metadata is expected here
58562024-02-07T18:56:55.039ZOn loop 26
58572024-02-07T18:56:55.039Z
58582024-02-07T18:56:55.039ZVerifying data now
58592024-02-07T18:57:00.242ZNo change after verify
58602024-02-07T18:57:00.242ZLoop: 26 Downstairs dump after verify (and repair):
58612024-02-07T18:57:00.256ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58622024-02-07T18:57:00.256Z 0 000-019 46 46 46 83 83 83 F F F
58632024-02-07T18:57:00.256Z 1 020-039 53 53 53 89 89 89 F F F
58642024-02-07T18:57:00.257Z 2 040-059 52 52 52 88 88 88 F F F
58652024-02-07T18:57:00.257Z 3 060-079 51 51 51 87 87 87 F F F
58662024-02-07T18:57:00.257Z 4 080-099 53 53 53 89 89 89 F F F
58672024-02-07T18:57:00.257Z 5 100-119 53 53 53 89 89 89 F F F
58682024-02-07T18:57:00.257Z 6 120-139 49 49 49 85 85 85 F F F
58692024-02-07T18:57:00.257Z 7 140-159 51 51 51 87 87 87 F F F
58702024-02-07T18:57:00.257Z 8 160-179 51 51 51 87 87 87 F F F
58712024-02-07T18:57:00.257Z 9 180-199 44 44 44 80 80 80 F F F
58722024-02-07T18:57:00.257Z 10 200-219 52 52 52 88 88 88 F F F
58732024-02-07T18:57:00.257Z 11 220-239 52 52 52 88 88 88 F F F
58742024-02-07T18:57:00.257Z 12 240-259 49 49 49 85 85 85 F F F
58752024-02-07T18:57:00.257Z 13 260-279 49 49 49 85 85 85 F F F
58762024-02-07T18:57:00.257Z 14 280-299 48 48 48 84 84 84 F F F
58772024-02-07T18:57:00.260Z 15 300-319 52 52 52 88 88 88 F F F
58782024-02-07T18:57:00.260Z 16 320-339 49 49 49 85 85 85 F F F
58792024-02-07T18:57:00.260Z 17 340-359 46 46 46 83 83 83 F F F
58802024-02-07T18:57:00.260Z 18 360-379 46 46 46 83 83 83 F F F
58812024-02-07T18:57:00.260Z 19 380-399 53 53 53 89 89 89 F F F
58822024-02-07T18:57:00.260Z 20 400-419 51 51 51 87 87 87 F F F
58832024-02-07T18:57:00.261Z 21 420-439 51 51 51 87 87 87 F F F
58842024-02-07T18:57:00.261Z 22 440-459 52 52 52 88 88 88 F F F
58852024-02-07T18:57:00.261Z 23 460-479 53 53 53 89 89 89 F F F
58862024-02-07T18:57:00.261Z 24 480-499 46 46 46 83 83 83 F F F
58872024-02-07T18:57:00.261Z 25 500-519 44 44 44 80 80 80 F F F
58882024-02-07T18:57:00.261Z 26 520-539 52 52 52 88 88 88 F F F
58892024-02-07T18:57:00.261Z 27 540-559 53 53 53 89 89 89 F F F
58902024-02-07T18:57:00.261Z 28 560-579 49 49 49 85 85 85 F F F
58912024-02-07T18:57:00.261Z 29 580-599 51 51 51 87 87 87 F F F
58922024-02-07T18:57:00.261ZMax gen: 53, Max flush: 89
58932024-02-07T18:57:00.261Z
58942024-02-07T18:57:00.261ZBegin loop 27
58952024-02-07T18:57:00.261ZDownstairs to restart: 1
58962024-02-07T18:57:00.281Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.531973808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351}
58972024-02-07T18:57:00.281Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.5323251Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351}
58982024-02-07T18:57:00.281Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532352766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351}
58992024-02-07T18:57:00.288Z{"msg":"Crucible 30e80a7b-f0d6-42b4-af94-3588abfe0f31 has session id: 32b4d8b1-1c3a-46c9-8b92-8822763c441b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532385392Z","hostname":"Crucible runtime is spawned
59002024-02-07T18:57:00.288Zip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59012024-02-07T18:57:00.288Z{"msg":"Upstairs opts: Upstairs UUID: 30e80a7b-f0d6-42b4-af94-3588abfe0f31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532442954Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59022024-02-07T18:57:00.288Z{"msg":"Crucible stats registered with UUID: 30e80a7b-f0d6-42b4-af94-3588abfe0f31","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532486688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59032024-02-07T18:57:00.288Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532510595Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351}
59042024-02-07T18:57:00.288Z{"msg":"30e80a7b-f0d6-42b4-af94-3588abfe0f31 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532721056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59052024-02-07T18:57:00.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532748443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59062024-02-07T18:57:00.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532765151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59072024-02-07T18:57:00.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532864437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59082024-02-07T18:57:00.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532906202Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59092024-02-07T18:57:00.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532938587Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59102024-02-07T18:57:00.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.532970233Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59112024-02-07T18:57:00.289Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533134061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59122024-02-07T18:57:00.289Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533174495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59132024-02-07T18:57:00.289Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533191053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59142024-02-07T18:57:00.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.5332168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59152024-02-07T18:57:00.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533232218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59162024-02-07T18:57:00.289Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:59.533256334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59172024-02-07T18:57:00.289Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:59.53328734Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59182024-02-07T18:57:00.289Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:56:59.533337013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59192024-02-07T18:57:00.289Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533374328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59202024-02-07T18:57:00.289Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533392666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59212024-02-07T18:57:00.289Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533428361Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59222024-02-07T18:57:00.289Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.5338743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59232024-02-07T18:57:00.289Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533926493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59242024-02-07T18:57:00.289Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.533955729Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59252024-02-07T18:57:00.289Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.534003803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59262024-02-07T18:57:00.289Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.534600912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59272024-02-07T18:57:00.289Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.534642537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59282024-02-07T18:57:00.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.535218039Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59292024-02-07T18:57:00.289Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.535255434Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59302024-02-07T18:57:00.289Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.535291239Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59312024-02-07T18:57:00.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.535313346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59322024-02-07T18:57:00.289Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.535327824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59332024-02-07T18:57:00.289Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:56:59.535348271Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59342024-02-07T18:57:01.282Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.534928978Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59352024-02-07T18:57:01.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.535162156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59362024-02-07T18:57:01.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.536143443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59372024-02-07T18:57:01.287Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.536178249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59382024-02-07T18:57:01.287Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.536737593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59392024-02-07T18:57:01.287Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537358119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59402024-02-07T18:57:01.287Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537395164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59412024-02-07T18:57:01.287Z{"msg":"[0]R flush_numbers[0..12]: [83, 89, 88, 87, 89, 89, 85, 87, 87, 80, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537411302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59422024-02-07T18:57:01.287Z{"msg":"[0]R generation[0..12]: [46, 53, 52, 51, 53, 53, 49, 51, 51, 44, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537433719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59432024-02-07T18:57:01.287Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537466335Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59442024-02-07T18:57:01.287Z{"msg":"[1]R flush_numbers[0..12]: [83, 89, 88, 87, 89, 89, 85, 87, 87, 80, 88, 88]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
59452024-02-07T18:57:01.287Z,"time":"2024-02-07T18:57:00.537483832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59462024-02-07T18:57:01.287Z{"msg":"[1]R generation[0..12]: [46, 53, 52, 51, 53, 53, 49, 51, 51, 44, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537512318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59472024-02-07T18:57:01.287Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537545314Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59482024-02-07T18:57:01.287Z{"msg":"[2]R flush_numbers[0..12]: [83, 89, 88, 87, 89, 89, 85, 87, 87, 80, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537578929Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59492024-02-07T18:57:01.288Z{"msg":"[2]R generation[0..12]: [46, 53, 52, 51, 53, 53, 49, 51, 51, 44, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537612775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59502024-02-07T18:57:01.288Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537631122Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59512024-02-07T18:57:01.288Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537662418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59522024-02-07T18:57:01.288Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537680975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59532024-02-07T18:57:01.288Z{"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537711091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59542024-02-07T18:57:01.288Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537727359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59552024-02-07T18:57:01.288Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537758635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59562024-02-07T18:57:01.288Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537776343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59572024-02-07T18:57:01.288Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537805889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59582024-02-07T18:57:01.288Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537821156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59592024-02-07T18:57:01.288Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537852192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59602024-02-07T18:57:01.288Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59612024-02-07T18:57:01.288Z{"msg":"30e80a7b-f0d6-42b4-af94-3588abfe0f31 is now active with session: 32b4d8b1-1c3a-46c9-8b92-8822763c441b","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
59622024-02-07T18:57:01.288Z,"time":"2024-02-07T18:57:00.537897776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59632024-02-07T18:57:01.288Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537913094Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351,"session_id":"32b4d8b1-1c3a-46c9-8b92-8822763c441b"}
59642024-02-07T18:57:01.288Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:00.537928352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351}
59652024-02-07T18:57:01.288ZLoading write count information from file "/tmp/test_repair_verify.data"
59662024-02-07T18:57:01.288ZRun Repair workload
59672024-02-07T18:57:01.292Z01/30 Write block 189 len 40960 data: 3 3 6 6 5 5 4 5 5 6
59682024-02-07T18:57:01.292Z02/30 Write block 36 len 24576 data: 7 4 3 3 5 4
59692024-02-07T18:57:01.292Z03/30 Read block 446 len 8192
59702024-02-07T18:57:01.292Z04/30 Write block 359 len 28672 data: 4 6 7 8 7 6 5
59712024-02-07T18:57:01.292Z05/30 Write block 410 len 8192 data: 5 6
59722024-02-07T18:57:01.292Z06/30 Read block 60 len 28672
59732024-02-07T18:57:01.292Z07/30 Read block 115 len 32768
59742024-02-07T18:57:01.296Z08/30 Write block 413 len 24576 data: 4 4 4 2 5 5
59752024-02-07T18:57:01.296Z09/30 Write block 445 len 28672 data: 6 6 4 4 4 4 5
59762024-02-07T18:57:01.296Z10/30 Read block 301 len 32768
59772024-02-07T18:57:01.296Z11/30 Read block 561 len 8192
59782024-02-07T18:57:01.296Z12/30 Flush
59792024-02-07T18:57:01.299Z13/30 Write block 411 len 4096 data: 7
59802024-02-07T18:57:01.299Z14/30 Read block 426 len 4096
59812024-02-07T18:57:01.299Z15/30 Write block 366 len 4096 data: 6
59822024-02-07T18:57:01.303Z16/30 Write block 410 len 36864 data: 6 8 4 5 5 5 3 6 6
59832024-02-07T18:57:01.303Z17/30 Write block 34 len 20480 data: 8 7 8 5 4
59842024-02-07T18:57:01.303Z18/30 Write block 96 len 40960 data: 6 6 5 4 3 3 3 3 6 7
59852024-02-07T18:57:01.303Z19/30 Read block 277 len 28672
59862024-02-07T18:57:01.307Z20/30 Read block 261 len 24576
59872024-02-07T18:57:01.307Z21/30 Write block 417 len 4096 data: 7
59882024-02-07T18:57:01.307Z22/30 Read block 462 len 32768
59892024-02-07T18:57:01.307Z23/30 Write block 535 len 12288 data: 7 8 9
59902024-02-07T18:57:01.310Z24/30 Write block 127 len 32768 data: 5 6 6 5 4 4 4 4
59912024-02-07T18:57:01.310Z25/30 Read block 565 len 32768
59922024-02-07T18:57:01.310Z26/30 Read block 576 len 24576
59932024-02-07T18:57:01.310Z27/30 Read block 241 len 8192
59942024-02-07T18:57:01.310Z28/30 Read block 446 len 12288
59952024-02-07T18:57:01.314Z29/30 Read block 56 len 40960
59962024-02-07T18:57:01.314Z30/30 Write block 349 len 32768 data: 6 7 7 4 3 5 5 3
59972024-02-07T18:57:01.314Z----------------------------------------------------------------
59982024-02-07T18:57:01.315Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59992024-02-07T18:57:01.315ZStates: Active Active Active
60002024-02-07T18:57:01.315ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
60012024-02-07T18:57:01.315Z 1 Acked 1000 Write 10 Done Done Done false
60022024-02-07T18:57:01.315Z 2 Acked 1001 Write 6 Done Done Done false
60032024-02-07T18:57:01.315Z 3 Acked 1002 Read 2 Done Done Done false
60042024-02-07T18:57:01.315Z 4 Acked 1003 Write 7 Done Done Done false
60052024-02-07T18:57:01.315Z 5 Acked 1004 Write 2 Done Sent Done false
60062024-02-07T18:57:01.315Z 6 Acked 1005 Read 7 Done Sent Done false
60072024-02-07T18:57:01.315Z 7 Acked 1006 Read 8 Done Sent Done false
60082024-02-07T18:57:01.315Z 8 Acked 1007 Write 6 Done Sent Done false
60092024-02-07T18:57:01.315Z 9 Acked 1008 Write 7 Done Sent Done false
60102024-02-07T18:57:01.315Z 10 Acked 1009 Read 8 Done Sent Done false
60112024-02-07T18:57:01.315Z 11 Acked 1010 Read 2 Done Sent Done false
60122024-02-07T18:57:01.315Z 12 Acked 1011 Flush 0 Done Sent Done false
60132024-02-07T18:57:01.315Z 13 Acked 1012 Write 1 Done Sent Done false
60142024-02-07T18:57:01.315Z 14 Acked 1013 Read 1 Done Sent Done false
60152024-02-07T18:57:01.315Z 15 Acked 1014 Write 1 Done Sent Done false
60162024-02-07T18:57:01.315Z 16 Acked 1015 Write 9 Done Sent Done false
60172024-02-07T18:57:01.315Z 17 Acked 1016 Write 5 Done Sent Done false
60182024-02-07T18:57:01.316Z 18 Acked 1017 Write 10 Done Sent Done false
60192024-02-07T18:57:01.316Z 19 Acked 1018 Read 7 Done Sent Done false
60202024-02-07T18:57:01.316Z 20 Acked 1019 Read 6 Done Sent Done false
60212024-02-07T18:57:01.316Z 21 Acked 1020 Write 1 Done Sent Done false
60222024-02-07T18:57:01.316Z 22 Acked 1021 Read 8 Done Sent Done false
60232024-02-07T18:57:01.316Z 23 Acked 1022 Write 3 Done Sent Done false
60242024-02-07T18:57:01.316Z 24 Acked 1023 Write 8 Done Sent Done false
60252024-02-07T18:57:01.316Z 25 Acked 1024 Read 8 Done Sent Done false
60262024-02-07T18:57:01.316Z 26 Acked 1025 Read 6 Done Sent Done false
60272024-02-07T18:57:01.316Z 27 Acked 1026 Read 2 Done Sent Done false
60282024-02-07T18:57:01.316Z 28 Acked 1027 Read 3 Done Sent Done false
60292024-02-07T18:57:01.316Z 29 Acked 1028 Read 10 Done Sent Done false
60302024-02-07T18:57:01.316Z 30 Acked 1029 Write 8 Sent Sent Sent false
60312024-02-07T18:57:01.316Z STATES DS:0 DS:1 DS:2 TOTAL
60322024-02-07T18:57:01.316Z New 0 0 0 0
60332024-02-07T18:57:01.316Z Sent 1 26 1 28
60342024-02-07T18:57:01.316Z Done 29 4 29 62
60352024-02-07T18:57:01.316Z Skipped 0 0 0 0
60362024-02-07T18:57:01.316Z Error 0 0 0 0
60372024-02-07T18:57:01.316ZLast Flush: 1011 0 1011
60382024-02-07T18:57:01.316ZDownstairs last five completed:
60392024-02-07T18:57:01.316ZUpstairs last five completed: 30 29 28 27 26
60402024-02-07T18:57:01.316Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:00.566837681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1351}
60412024-02-07T18:57:01.317ZWrote out file "/tmp/test_repair_verify.data"
60422024-02-07T18:57:01.320Z
60432024-02-07T18:57:01.323ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
60442024-02-07T18:57:01.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60452024-02-07T18:57:01.341Z 0 000-019 46 46 46 83 83 83 F F F
60462024-02-07T18:57:01.341Z 1 020-039 54 53 54 90 89 90 T T T <---
60472024-02-07T18:57:01.341Z 2 040-059 54 52 54 90 88 90 F T F <---
60482024-02-07T18:57:01.341Z 3 060-079 51 51 51 87 87 87 F F F
60492024-02-07T18:57:01.341Z 4 080-099 53 53 53 89 89 89 T F T <---
60502024-02-07T18:57:01.341Z 5 100-119 53 53 53 89 89 89 T F T <---
60512024-02-07T18:57:01.341Z 6 120-139 49 49 49 85 85 85 T F T <---
60522024-02-07T18:57:01.341Z 7 140-159 51 51 51 87 87 87 F F F
60532024-02-07T18:57:01.341Z 8 160-179 51 51 51 87 87 87 F F F
60542024-02-07T18:57:01.341Z 9 180-199 54 44 54 90 80 90 F T F <---
60552024-02-07T18:57:01.341Z 10 200-219 52 52 52 88 88 88 F F F
60562024-02-07T18:57:01.341Z 11 220-239 52 52 52 88 88 88 F F F
60572024-02-07T18:57:01.341Z 12 240-259 49 49 49 85 85 85 F F F
60582024-02-07T18:57:01.341Z 13 260-279 49 49 49 85 85 85 F F F
60592024-02-07T18:57:01.342Z 14 280-299 48 48 48 84 84 84 F F F
60602024-02-07T18:57:01.342Z 15 300-319 52 52 52 88 88 88 F F F
60612024-02-07T18:57:01.345Z 16 320-339 49 49 49 85 85 85 F F F
60622024-02-07T18:57:01.345Z 17 340-359 54 46 54 90 83 90 T T T <---
60632024-02-07T18:57:01.345Z 18 360-379 54 46 54 90 83 90 T T T <---
60642024-02-07T18:57:01.345Z 19 380-399 53 53 53 89 89 89 F F F
60652024-02-07T18:57:01.345Z 20 400-419 54 51 54 90 87 90 T F T <---
60662024-02-07T18:57:01.345Z 21 420-439 51 51 51 87 87 87 F F F
60672024-02-07T18:57:01.345Z 22 440-459 54 52 54 90 88 90 F F F <---
60682024-02-07T18:57:01.345Z 23 460-479 53 53 53 89 89 89 F F F
60692024-02-07T18:57:01.346Z 24 480-499 46 46 46 83 83 83 F F F
60702024-02-07T18:57:01.346Z 25 500-519 44 44 44 80 80 80 F F F
60712024-02-07T18:57:01.346Z 26 520-539 52 52 52 88 88 88 T F T <---
60722024-02-07T18:57:01.346Z 27 540-559 53 53 53 89 89 89 F F F
60732024-02-07T18:57:01.346Z 28 560-579 49 49 49 85 85 85 F F F
60742024-02-07T18:57:01.346Z 29 580-599 51 51 51 87 87 87 F F F
60752024-02-07T18:57:01.346ZMax gen: 54, Max flush: 90
60762024-02-07T18:57:01.346ZError: Difference in extent metadata found!
60772024-02-07T18:57:01.346ZA Difference in extent metadata is expected here
60782024-02-07T18:57:01.346ZOn loop 27
60792024-02-07T18:57:01.346Z
60802024-02-07T18:57:01.350ZVerifying data now
60812024-02-07T18:57:06.911ZNo change after verify
60822024-02-07T18:57:06.912ZLoop: 27 Downstairs dump after verify (and repair):
60832024-02-07T18:57:06.927ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60842024-02-07T18:57:06.927Z 0 000-019 46 46 46 83 83 83 F F F
60852024-02-07T18:57:06.927Z 1 020-039 55 55 55 91 91 91 F F F
60862024-02-07T18:57:06.927Z 2 040-059 54 54 54 90 90 90 F F F
60872024-02-07T18:57:06.927Z 3 060-079 51 51 51 87 87 87 F F F
60882024-02-07T18:57:06.927Z 4 080-099 55 55 55 91 91 91 F F F
60892024-02-07T18:57:06.927Z 5 100-119 55 55 55 91 91 91 F F F
60902024-02-07T18:57:06.927Z 6 120-139 55 55 55 91 91 91 F F F
60912024-02-07T18:57:06.927Z 7 140-159 51 51 51 87 87 87 F F F
60922024-02-07T18:57:06.927Z 8 160-179 51 51 51 87 87 87 F F F
60932024-02-07T18:57:06.927Z 9 180-199 54 54 54 90 90 90 F F F
60942024-02-07T18:57:06.931Z 10 200-219 52 52 52 88 88 88 F F F
60952024-02-07T18:57:06.931Z 11 220-239 52 52 52 88 88 88 F F F
60962024-02-07T18:57:06.931Z 12 240-259 49 49 49 85 85 85 F F F
60972024-02-07T18:57:06.931Z 13 260-279 49 49 49 85 85 85 F F F
60982024-02-07T18:57:06.931Z 14 280-299 48 48 48 84 84 84 F F F
60992024-02-07T18:57:06.931Z 15 300-319 52 52 52 88 88 88 F F F
61002024-02-07T18:57:06.931Z 16 320-339 49 49 49 85 85 85 F F F
61012024-02-07T18:57:06.931Z 17 340-359 55 55 55 91 91 91 F F F
61022024-02-07T18:57:06.931Z 18 360-379 55 55 55 91 91 91 F F F
61032024-02-07T18:57:06.931Z 19 380-399 53 53 53 89 89 89 F F F
61042024-02-07T18:57:06.931Z 20 400-419 55 55 55 91 91 91 F F F
61052024-02-07T18:57:06.931Z 21 420-439 51 51 51 87 87 87 F F F
61062024-02-07T18:57:06.931Z 22 440-459 54 54 54 90 90 90 F F F
61072024-02-07T18:57:06.931Z 23 460-479 53 53 53 89 89 89 F F F
61082024-02-07T18:57:06.931Z 24 480-499 46 46 46 83 83 83 F F F
61092024-02-07T18:57:06.931Z 25 500-519 44 44 44 80 80 80 F F F
61102024-02-07T18:57:06.931Z 26 520-539 55 55 55 91 91 91 F F F
61112024-02-07T18:57:06.932Z 27 540-559 53 53 53 89 89 89 F F F
61122024-02-07T18:57:06.932Z 28 560-579 49 49 49 85 85 85 F F F
61132024-02-07T18:57:06.932Z 29 580-599 51 51 51 87 87 87 F F F
61142024-02-07T18:57:06.932ZMax gen: 55, Max flush: 91
61152024-02-07T18:57:06.932Z
61162024-02-07T18:57:06.932ZBegin loop 28
61172024-02-07T18:57:06.932ZDownstairs to restart: 1
61182024-02-07T18:57:06.949Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200040637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359}
61192024-02-07T18:57:06.949Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200275466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359}
61202024-02-07T18:57:06.949Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200300002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359Crucible runtime is spawned
61212024-02-07T18:57:06.949Z}
61222024-02-07T18:57:06.955Z{"msg":"Crucible 5b3340ff-3049-4c52-9eca-17b827271fbe has session id: 9953b6f6-664d-4dd5-8a4f-9ab62c0d422e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200340037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61232024-02-07T18:57:06.955Z{"msg":"Upstairs opts: Upstairs UUID: 5b3340ff-3049-4c52-9eca-17b827271fbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200386241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61242024-02-07T18:57:06.955Z{"msg":"Crucible stats registered with UUID: 5b3340ff-3049-4c52-9eca-17b827271fbe","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200413377Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61252024-02-07T18:57:06.955Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200444823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359}
61262024-02-07T18:57:06.955Z{"msg":"5b3340ff-3049-4c52-9eca-17b827271fbe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200637107Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61272024-02-07T18:57:06.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200673542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61282024-02-07T18:57:06.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.20068964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61292024-02-07T18:57:06.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200716556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61302024-02-07T18:57:06.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200738863Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61312024-02-07T18:57:06.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.20075762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61322024-02-07T18:57:06.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200782727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61332024-02-07T18:57:06.956Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200824101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61342024-02-07T18:57:06.956Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200856277Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61352024-02-07T18:57:06.956Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.200882494Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61362024-02-07T18:57:06.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201073798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61372024-02-07T18:57:06.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201123871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61382024-02-07T18:57:06.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201145108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61392024-02-07T18:57:06.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201856672Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61402024-02-07T18:57:06.956Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201893687Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61412024-02-07T18:57:06.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201909135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61422024-02-07T18:57:06.956Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.201924502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61432024-02-07T18:57:06.956Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.202445972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61442024-02-07T18:57:06.956Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.202571115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61452024-02-07T18:57:06.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203029063Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61462024-02-07T18:57:06.956Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203081446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61472024-02-07T18:57:06.956Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203112002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61482024-02-07T18:57:06.956Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203144168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61492024-02-07T18:57:06.956Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203178993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61502024-02-07T18:57:06.956Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.20320458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61512024-02-07T18:57:06.956Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203220137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61522024-02-07T18:57:06.956Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203234365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61532024-02-07T18:57:06.956Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.203677345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61542024-02-07T18:57:06.957Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204119726Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61552024-02-07T18:57:06.957Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204168689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61562024-02-07T18:57:06.957Z{"msg":"[0]R flush_numbers[0..12]: [83, 91, 90, 87, 91, 91, 91, 87, 87, 90, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204187607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61572024-02-07T18:57:06.957Z{"msg":"[0]R generation[0..12]: [46, 55, 54, 51, 55, 55, 55, 51, 51, 54, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204211243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61582024-02-07T18:57:06.957Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204226381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61592024-02-07T18:57:06.957Z{"msg":"[1]R flush_numbers[0..12]: [83, 91, 90, 87, 91, 91, 91, 87, 87, 90, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204247019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61602024-02-07T18:57:06.957Z{"msg":"[1]R generation[0..12]: [46, 55, 54, 51, 55, 55, 55, 51, 51, 54, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204261346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61612024-02-07T18:57:06.957Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","vWait for a query_work_queue command to finish before sending IO
61622024-02-07T18:57:06.957Z":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204286573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61632024-02-07T18:57:06.957Z{"msg":"[2]R flush_numbers[0..12]: [83, 91, 90, 87, 91, 91, 91, 87, 87, 90, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204301561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61642024-02-07T18:57:06.957Z{"msg":"[2]R generation[0..12]: [46, 55, 54, 51, 55, 55, 55, 51, 51, 54, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204322078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61652024-02-07T18:57:06.957Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204336846Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61662024-02-07T18:57:06.957Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204357524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61672024-02-07T18:57:06.957Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204371572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61682024-02-07T18:57:06.957Z{"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204391679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61692024-02-07T18:57:06.957Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204405867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61702024-02-07T18:57:06.957Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204440582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61712024-02-07T18:57:06.957Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204462669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61722024-02-07T18:57:06.957Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204495305Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61732024-02-07T18:57:06.957Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204512612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61742024-02-07T18:57:06.957Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204546898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61752024-02-07T18:57:06.957Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204582093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61762024-02-07T18:57:06.957Z{"msg":"5b3340ff-3049-4c52-9eca-17b827271fbe is now active with session: 9953b6f6-664d-4dd5-8a4f-9ab62c0d422e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.20460782Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61772024-02-07T18:57:06.958Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204639835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359,"session_id":"9953b6f6-664d-4dd5-8a4f-9ab62c0d422e"}
61782024-02-07T18:57:06.958Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:06.204672481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359}
61792024-02-07T18:57:06.958ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61802024-02-07T18:57:06.958ZLoading write count information from file "/tmp/test_repair_verify.data"
61812024-02-07T18:57:06.958ZRun Repair workload
61822024-02-07T18:57:06.962Z01/30 Write block 188 len 4096 data: 4
61832024-02-07T18:57:06.962Z02/30 Write block 148 len 8192 data: 8 11
61842024-02-07T18:57:06.962Z03/30 Read block 514 len 4096
61852024-02-07T18:57:06.962Z04/30 Write block 286 len 4096 data: 7
61862024-02-07T18:57:06.962Z05/30 Flush
61872024-02-07T18:57:06.962Z06/30 Write block 202 len 4096 data: 4
61882024-02-07T18:57:06.962Z07/30 Flush
61892024-02-07T18:57:06.962Z08/30 Flush
61902024-02-07T18:57:06.962Z09/30 Write block 198 len 12288 data: 7 4 3
61912024-02-07T18:57:06.962Z10/30 Write block 117 len 28672 data: 6 7 8 7 7 8 6
61922024-02-07T18:57:06.965Z11/30 Write block 599 len 4096 data: 2
61932024-02-07T18:57:06.965Z12/30 Write block 16 len 8192 data: 7 8
61942024-02-07T18:57:06.965Z13/30 Write block 24 len 4096 data: 9
61952024-02-07T18:57:06.965Z14/30 Read block 321 len 40960
61962024-02-07T18:57:06.965Z15/30 Read block 21 len 28672
61972024-02-07T18:57:06.969Z16/30 Write block 12 len 28672 data: 4 5 6 5 8 9 7
61982024-02-07T18:57:06.969Z17/30 Read block 34 len 36864
61992024-02-07T18:57:06.969Z18/30 Write block 406 len 20480 data: 3 3 3 4 7
62002024-02-07T18:57:06.971Z19/30 Write block 452 len 28672 data: 4 5 4 3 2 3 4
62012024-02-07T18:57:06.971Z20/30 Read block 11 len 12288
62022024-02-07T18:57:06.971Z21/30 Read block 412 len 32768
62032024-02-07T18:57:06.972Z22/30 Read block 412 len 32768
62042024-02-07T18:57:06.974Z23/30 Read block 61 len 36864
62052024-02-07T18:57:06.975Z24/30 Write block 349 len 8192 data: 7 8
62062024-02-07T18:57:06.975Z25/30 Write block 82 len 24576 data: 3 4 3 2 3 3
62072024-02-07T18:57:06.975Z26/30 Read block 76 len 16384
62082024-02-07T18:57:06.978Z27/30 Read block 63 len 24576
62092024-02-07T18:57:06.978Z28/30 Write block 47 len 8192 data: 5 6
62102024-02-07T18:57:06.979Z29/30 Write block 313 len 20480 data: 3 4 6 8 6
62112024-02-07T18:57:06.979Z30/30 Read block 79 len 24576
62122024-02-07T18:57:06.982Z----------------------------------------------------------------
62132024-02-07T18:57:06.982Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
62142024-02-07T18:57:06.982ZStates: Active Active Active
62152024-02-07T18:57:06.982ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
62162024-02-07T18:57:06.982Z 1 Acked 1000 Write 1 Done Done Done false
62172024-02-07T18:57:06.982Z 2 Acked 1001 Write 2 Done Done Done false
62182024-02-07T18:57:06.982Z 3 Acked 1002 Read 1 Done Sent Done false
62192024-02-07T18:57:06.982Z 4 Acked 1003 Write 1 Done Sent Done false
62202024-02-07T18:57:06.982Z 5 Acked 1004 Flush 0 Done Sent Done false
62212024-02-07T18:57:06.982Z 6 Acked 1005 Write 1 Done Sent Done false
62222024-02-07T18:57:06.982Z 7 Acked 1006 Flush 0 Done Sent Done false
62232024-02-07T18:57:06.982Z 8 Acked 1007 Flush 0 Done Sent Done false
62242024-02-07T18:57:06.982Z 9 Acked 1008 Write 3 Done Sent Done false
62252024-02-07T18:57:06.982Z 10 Acked 1009 Write 7 Done Sent Done false
62262024-02-07T18:57:06.982Z 11 Acked 1010 Write 1 Done Sent Done false
62272024-02-07T18:57:06.982Z 12 Acked 1011 Write 2 Done Sent Done false
62282024-02-07T18:57:06.983Z 13 Acked 1012 Write 1 Done Sent Done false
62292024-02-07T18:57:06.983Z 14 Acked 1013 Read 10 Done Sent Done false
62302024-02-07T18:57:06.983Z 15 Acked 1014 Read 7 Done Sent Done false
62312024-02-07T18:57:06.983Z 16 Acked 1015 Write 7 Done Sent Done false
62322024-02-07T18:57:06.983Z 17 Acked 1016 Read 9 Done Sent Done false
62332024-02-07T18:57:06.983Z 18 Acked 1017 Write 5 Done Sent Done false
62342024-02-07T18:57:06.983Z 19 Acked 1018 Write 7 Done Sent Done false
62352024-02-07T18:57:06.983Z 20 Acked 1019 Read 3 Done Sent Done false
62362024-02-07T18:57:06.983Z 21 Acked 1020 Read 8 Done Sent Done false
62372024-02-07T18:57:06.983Z 22 Acked 1021 Read 8 Done Sent Done false
62382024-02-07T18:57:06.983Z 23 Acked 1022 Read 9 Done Sent Done false
62392024-02-07T18:57:06.983Z 24 Acked 1023 Write 2 Done Sent Done false
62402024-02-07T18:57:06.983Z 25 Acked 1024 Write 6 Done Sent Done false
62412024-02-07T18:57:06.983Z 26 Acked 1025 Read 4 Done Sent Done false
62422024-02-07T18:57:06.983Z 27 Acked 1026 Read 6 Done Sent Done false
62432024-02-07T18:57:06.983Z 28 Acked 1027 Write 2 Done Sent Done false
62442024-02-07T18:57:06.983Z 29 Acked 1028 Write 5 Done Sent Done false
62452024-02-07T18:57:06.983Z 30 Acked 1029 Read 6 Sent Sent Done false
62462024-02-07T18:57:06.983Z STATES DS:0 DS:1 DS:2 TOTAL
62472024-02-07T18:57:06.983Z New 0 0 0 0
62482024-02-07T18:57:06.983Z Sent 1 28 0 29
62492024-02-07T18:57:06.983Z Done 29 2 30 61
62502024-02-07T18:57:06.983Z Skipped 0 0 0 0
62512024-02-07T18:57:06.983Z Error 0 0 0 0
62522024-02-07T18:57:06.984ZLast Flush: 1007 0 1007
62532024-02-07T18:57:06.984ZDownstairs last five completed:
62542024-02-07T18:57:06.984ZUpstairs last five completed: 30 29 28 27 26
62552024-02-07T18:57:06.984Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:06.231801472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1359}
62562024-02-07T18:57:06.984ZWrote out file "/tmp/test_repair_verify.data"
62572024-02-07T18:57:06.987Z
62582024-02-07T18:57:06.987ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
62592024-02-07T18:57:07.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62602024-02-07T18:57:07.006Z 0 000-019 46 46 46 83 83 83 T F T <---
62612024-02-07T18:57:07.007Z 1 020-039 55 55 55 91 91 91 T F T <---
62622024-02-07T18:57:07.007Z 2 040-059 54 54 54 90 90 90 T F T <---
62632024-02-07T18:57:07.007Z 3 060-079 51 51 51 87 87 87 F F F
62642024-02-07T18:57:07.007Z 4 080-099 55 55 55 91 91 91 T F T <---
62652024-02-07T18:57:07.007Z 5 100-119 55 55 55 91 91 91 T F T <---
62662024-02-07T18:57:07.007Z 6 120-139 55 55 55 91 91 91 T F T <---
62672024-02-07T18:57:07.007Z 7 140-159 56 51 56 92 87 92 F T F <---
62682024-02-07T18:57:07.007Z 8 160-179 51 51 51 87 87 87 F F F
62692024-02-07T18:57:07.007Z 9 180-199 56 54 56 92 90 92 T T T <---
62702024-02-07T18:57:07.010Z 10 200-219 56 52 56 93 88 93 T F T <---
62712024-02-07T18:57:07.010Z 11 220-239 52 52 52 88 88 88 F F F
62722024-02-07T18:57:07.010Z 12 240-259 49 49 49 85 85 85 F F F
62732024-02-07T18:57:07.011Z 13 260-279 49 49 49 85 85 85 F F F
62742024-02-07T18:57:07.011Z 14 280-299 56 48 56 92 84 92 F F F <---
62752024-02-07T18:57:07.011Z 15 300-319 52 52 52 88 88 88 T F T <---
62762024-02-07T18:57:07.011Z 16 320-339 49 49 49 85 85 85 F F F
62772024-02-07T18:57:07.011Z 17 340-359 55 55 55 91 91 91 T F T <---
62782024-02-07T18:57:07.011Z 18 360-379 55 55 55 91 91 91 F F F
62792024-02-07T18:57:07.011Z 19 380-399 53 53 53 89 89 89 F F F
62802024-02-07T18:57:07.011Z 20 400-419 55 55 55 91 91 91 T F T <---
62812024-02-07T18:57:07.011Z 21 420-439 51 51 51 87 87 87 F F F
62822024-02-07T18:57:07.011Z 22 440-459 54 54 54 90 90 90 T F T <---
62832024-02-07T18:57:07.011Z 23 460-479 53 53 53 89 89 89 F F F
62842024-02-07T18:57:07.011Z 24 480-499 46 46 46 83 83 83 F F F
62852024-02-07T18:57:07.011Z 25 500-519 44 44 44 80 80 80 F F F
62862024-02-07T18:57:07.011Z 26 520-539 55 55 55 91 91 91 F F F
62872024-02-07T18:57:07.011Z 27 540-559 53 53 53 89 89 89 F F F
62882024-02-07T18:57:07.011Z 28 560-579 49 49 49 85 85 85 F F F
62892024-02-07T18:57:07.011Z 29 580-599 51 51 51 87 87 87 T F T <---
62902024-02-07T18:57:07.011ZMax gen: 56, Max flush: 93
62912024-02-07T18:57:07.011ZError: Difference in extent metadata found!
62922024-02-07T18:57:07.011ZA Difference in extent metadata is expected here
62932024-02-07T18:57:07.011ZOn loop 28
62942024-02-07T18:57:07.011Z
62952024-02-07T18:57:07.015ZVerifying data now
62962024-02-07T18:57:12.757ZNo change after verify
62972024-02-07T18:57:12.758ZLoop: 28 Downstairs dump after verify (and repair):
62982024-02-07T18:57:12.773ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62992024-02-07T18:57:12.773Z 0 000-019 57 57 57 94 94 94 F F F
63002024-02-07T18:57:12.773Z 1 020-039 57 57 57 94 94 94 F F F
63012024-02-07T18:57:12.773Z 2 040-059 57 57 57 94 94 94 F F F
63022024-02-07T18:57:12.773Z 3 060-079 51 51 51 87 87 87 F F F
63032024-02-07T18:57:12.773Z 4 080-099 57 57 57 94 94 94 F F F
63042024-02-07T18:57:12.773Z 5 100-119 57 57 57 94 94 94 F F F
63052024-02-07T18:57:12.773Z 6 120-139 57 57 57 94 94 94 F F F
63062024-02-07T18:57:12.773Z 7 140-159 56 56 56 92 92 92 F F F
63072024-02-07T18:57:12.773Z 8 160-179 51 51 51 87 87 87 F F F
63082024-02-07T18:57:12.773Z 9 180-199 57 57 57 94 94 94 F F F
63092024-02-07T18:57:12.773Z 10 200-219 57 57 57 94 94 94 F F F
63102024-02-07T18:57:12.773Z 11 220-239 52 52 52 88 88 88 F F F
63112024-02-07T18:57:12.773Z 12 240-259 49 49 49 85 85 85 F F F
63122024-02-07T18:57:12.773Z 13 260-279 49 49 49 85 85 85 F F F
63132024-02-07T18:57:12.778Z 14 280-299 56 56 56 92 92 92 F F F
63142024-02-07T18:57:12.779Z 15 300-319 57 57 57 94 94 94 F F F
63152024-02-07T18:57:12.779Z 16 320-339 49 49 49 85 85 85 F F F
63162024-02-07T18:57:12.779Z 17 340-359 57 57 57 94 94 94 F F F
63172024-02-07T18:57:12.779Z 18 360-379 55 55 55 91 91 91 F F F
63182024-02-07T18:57:12.779Z 19 380-399 53 53 53 89 89 89 F F F
63192024-02-07T18:57:12.779Z 20 400-419 57 57 57 94 94 94 F F F
63202024-02-07T18:57:12.779Z 21 420-439 51 51 51 87 87 87 F F F
63212024-02-07T18:57:12.779Z 22 440-459 57 57 57 94 94 94 F F F
63222024-02-07T18:57:12.779Z 23 460-479 53 53 53 89 89 89 F F F
63232024-02-07T18:57:12.779Z 24 480-499 46 46 46 83 83 83 F F F
63242024-02-07T18:57:12.779Z 25 500-519 44 44 44 80 80 80 F F F
63252024-02-07T18:57:12.779Z 26 520-539 55 55 55 91 91 91 F F F
63262024-02-07T18:57:12.779Z 27 540-559 53 53 53 89 89 89 F F F
63272024-02-07T18:57:12.779Z 28 560-579 49 49 49 85 85 85 F F F
63282024-02-07T18:57:12.779Z 29 580-599 57 57 57 94 94 94 F F F
63292024-02-07T18:57:12.779ZMax gen: 57, Max flush: 94
63302024-02-07T18:57:12.779Z
63312024-02-07T18:57:12.779ZBegin loop 29
63322024-02-07T18:57:12.779ZDownstairs to restart: 2
63332024-02-07T18:57:12.796Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.04687624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367}
63342024-02-07T18:57:12.796Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047174889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367}
63352024-02-07T18:57:12.802Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047223793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367}
63362024-02-07T18:57:12.802Z{"msg":"Crucible b49fd3ef-da3e-48f5-91f4-2a1a27419e07 has session id: 220c76a6-801e-40c4-829a-4c4e057fc9f7","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047265437Z","hostname":"Crucible runtime is spawned
63372024-02-07T18:57:12.802Zip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63382024-02-07T18:57:12.802Z{"msg":"Upstairs opts: Upstairs UUID: b49fd3ef-da3e-48f5-91f4-2a1a27419e07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047334068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63392024-02-07T18:57:12.802Z{"msg":"Crucible stats registered with UUID: b49fd3ef-da3e-48f5-91f4-2a1a27419e07","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047379201Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63402024-02-07T18:57:12.802Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047406388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367}
63412024-02-07T18:57:12.802Z{"msg":"b49fd3ef-da3e-48f5-91f4-2a1a27419e07 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047505644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63422024-02-07T18:57:12.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047535531Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63432024-02-07T18:57:12.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047561927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63442024-02-07T18:57:12.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047597982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63452024-02-07T18:57:12.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047626138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63462024-02-07T18:57:12.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047662273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63472024-02-07T18:57:12.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.04768399Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63482024-02-07T18:57:12.802Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047717286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63492024-02-07T18:57:12.802Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047742052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63502024-02-07T18:57:12.802Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047765039Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63512024-02-07T18:57:12.802Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:12.047806424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63522024-02-07T18:57:12.802Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:12.04783621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63532024-02-07T18:57:12.802Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:12.047875394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63542024-02-07T18:57:12.803Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047911969Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63552024-02-07T18:57:12.803Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047949015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63562024-02-07T18:57:12.803Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.047987629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63572024-02-07T18:57:12.803Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.048043802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63582024-02-07T18:57:12.803Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.048079477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63592024-02-07T18:57:12.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.04864615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63602024-02-07T18:57:12.803Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.048685905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63612024-02-07T18:57:12.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.048712281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63622024-02-07T18:57:12.803Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.048744237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63632024-02-07T18:57:12.803Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.049292703Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63642024-02-07T18:57:12.803Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.049329408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63652024-02-07T18:57:12.803Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.049857516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63662024-02-07T18:57:12.803Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.049916518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63672024-02-07T18:57:12.803Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.050003757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63682024-02-07T18:57:12.803Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.050038792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63692024-02-07T18:57:12.803Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.05005488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63702024-02-07T18:57:12.803Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:12.050077537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63712024-02-07T18:57:13.797Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.048515308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63722024-02-07T18:57:13.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.048634471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63732024-02-07T18:57:13.802Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.049553907Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63742024-02-07T18:57:13.802Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.049594351Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63752024-02-07T18:57:13.802Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050083295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63762024-02-07T18:57:13.802Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.05071425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63772024-02-07T18:57:13.802Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050758104Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63782024-02-07T18:57:13.802Z{"msg":"[0]R flush_numbers[0..12]: [94, 94, 94, 87, 94, 94, 94, 92, 87, 94, 94, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.05078752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63792024-02-07T18:57:13.803Z{"msg":"[0]R generation[0..12]: [57, 57, 57, 51, 57, 57, 57, 56, 51, 57, 57, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050809987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63802024-02-07T18:57:13.803Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050841813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63812024-02-07T18:57:13.803Z{"msg":"[1]R flush_numbers[0..12]: [94, 94, 94, 87, 94, 94, 94, 92, 87, 94, 94, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050856961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63822024-02-07T18:57:13.803Z{"msg":"[1]R generation[0..12]: [57, 57, 57, 51, 57, 57, 57, 56, 51, 57, 57, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050870879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63832024-02-07T18:57:13.803Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050892606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63842024-02-07T18:57:13.803Z{"msg":"[2]R flush_numbers[0..12]: [94, 94, 94, 87, 94, 94, 94, 92, 87, 94, 94, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050907834Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63852024-02-07T18:57:13.803Z{"msg":"[2]R generation[0..12]: [57, 57, 57, 51, 57, 57, 57, 56, 51, 57, 57, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050922912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63862024-02-07T18:57:13.803Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050945039Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63872024-02-07T18:57:13.803Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050959047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
63882024-02-07T18:57:13.803Zdownstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63892024-02-07T18:57:13.803Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.050988423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63902024-02-07T18:57:13.803Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051001621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63912024-02-07T18:57:13.803Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051022518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63922024-02-07T18:57:13.803Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051036427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63932024-02-07T18:57:13.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051061653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63942024-02-07T18:57:13.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051093979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63952024-02-07T18:57:13.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051110497Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63962024-02-07T18:57:13.803Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051151021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63972024-02-07T18:57:13.803Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051177477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63982024-02-07T18:57:13.803Z{"msg":"b49fd3ef-da3e-48f5-91f4-2a1a27419e07 is now active with session: 220c76a6-801e-40c4-829a-4c4e057fc9f7","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051201464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
63992024-02-07T18:57:13.803Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051237569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367,"session_id":"220c76a6-801e-40c4-829a-4c4e057fc9f7"}
64002024-02-07T18:57:13.803Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:13.051259776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367}
64012024-02-07T18:57:13.803ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64022024-02-07T18:57:13.803ZLoading write count information from file "/tmp/test_repair_verify.data"
64032024-02-07T18:57:13.803ZRun Repair workload
64042024-02-07T18:57:13.804Z01/30 Write block 425 len 4096 data: 7
64052024-02-07T18:57:13.807Z02/30 Write block 240 len 28672 data: 7 7 7 8 7 5 5
64062024-02-07T18:57:13.807Z03/30 Write block 110 len 20480 data: 7 7 5 4 3
64072024-02-07T18:57:13.807Z04/30 Write block 511 len 32768 data: 5 7 6 6 5 4 3 4
64082024-02-07T18:57:13.807Z05/30 Read block 55 len 24576
64092024-02-07T18:57:13.807Z06/30 Read block 481 len 40960
64102024-02-07T18:57:13.807Z07/30 Read block 82 len 40960
64112024-02-07T18:57:13.810Z08/30 Read block 591 len 8192
64122024-02-07T18:57:13.810Z09/30 Write block 77 len 40960 data: 4 6 5 4 3 4 5 4 3 4
64132024-02-07T18:57:13.810Z10/30 Write block 453 len 32768 data: 6 5 4 3 4 5 4 3
64142024-02-07T18:57:13.810Z11/30 Read block 479 len 12288
64152024-02-07T18:57:13.813Z12/30 Read block 511 len 16384
64162024-02-07T18:57:13.813Z13/30 Read block 206 len 28672
64172024-02-07T18:57:13.813Z14/30 Write block 315 len 20480 data: 7 9 7 7 6
64182024-02-07T18:57:13.813Z15/30 Read block 56 len 40960
64192024-02-07T18:57:13.816Z16/30 Write block 232 len 24576 data: 8 8 9 12 10 9
64202024-02-07T18:57:13.816Z17/30 Read block 250 len 20480
64212024-02-07T18:57:13.819Z18/30 Read block 252 len 8192
64222024-02-07T18:57:13.819Z19/30 Write block 322 len 20480 data: 6 6 5 4 3
64232024-02-07T18:57:13.819Z20/30 Read block 392 len 12288
64242024-02-07T18:57:13.819Z21/30 Read block 120 len 20480
64252024-02-07T18:57:13.822Z22/30 Write block 406 len 32768 data: 4 4 4 5 8 9 5 6
64262024-02-07T18:57:13.822Z23/30 Write block 3 len 20480 data: 3 3 4 3 4
64272024-02-07T18:57:13.822Z24/30 Read block 60 len 40960
64282024-02-07T18:57:13.825Z25/30 Write block 116 len 24576 data: 5 7 8 9 8 8
64292024-02-07T18:57:13.825Z26/30 Write block 524 len 28672 data: 4 3 4 5 5 5 5
64302024-02-07T18:57:13.825Z27/30 Write block 413 len 24576 data: 7 6 6 4 8 7
64312024-02-07T18:57:13.825Z28/30 Read block 8 len 20480
64322024-02-07T18:57:13.828Z29/30 Write block 5 len 32768 data: 5 4 5 4 5 4 4 5
64332024-02-07T18:57:13.828Z30/30 Read block 247 len 28672
64342024-02-07T18:57:13.833Z----------------------------------------------------------------
64352024-02-07T18:57:13.833Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
64362024-02-07T18:57:13.833ZStates: Active Active Active
64372024-02-07T18:57:13.834ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
64382024-02-07T18:57:13.834Z 1 Acked 1000 Write 1 Done Done Done false
64392024-02-07T18:57:13.834Z 2 Acked 1001 Write 7 Done Done Done false
64402024-02-07T18:57:13.834Z 3 Acked 1002 Write 5 Done Done Sent false
64412024-02-07T18:57:13.834Z 4 Acked 1003 Write 8 Done Done Sent false
64422024-02-07T18:57:13.834Z 5 Acked 1004 Read 6 Done Done Sent false
64432024-02-07T18:57:13.834Z 6 Acked 1005 Read 10 Done Done Sent false
64442024-02-07T18:57:13.834Z 7 Acked 1006 Read 10 Done Done Sent false
64452024-02-07T18:57:13.834Z 8 Acked 1007 Read 2 Done Done Sent false
64462024-02-07T18:57:13.834Z 9 Acked 1008 Write 10 Done Done Sent false
64472024-02-07T18:57:13.834Z 10 Acked 1009 Write 8 Done Done Sent false
64482024-02-07T18:57:13.834Z 11 Acked 1010 Read 3 Done Done Sent false
64492024-02-07T18:57:13.834Z 12 Acked 1011 Read 4 Done Done Sent false
64502024-02-07T18:57:13.834Z 13 Acked 1012 Read 7 Done Done Sent false
64512024-02-07T18:57:13.834Z 14 Acked 1013 Write 5 Done Done Sent false
64522024-02-07T18:57:13.834Z 15 Acked 1014 Read 10 Done Done Sent false
64532024-02-07T18:57:13.834Z 16 Acked 1015 Write 6 Done Done Sent false
64542024-02-07T18:57:13.834Z 17 Acked 1016 Read 5 Done Done Sent false
64552024-02-07T18:57:13.834Z 18 Acked 1017 Read 2 Done Done Sent false
64562024-02-07T18:57:13.834Z 19 Acked 1018 Write 5 Done Done Sent false
64572024-02-07T18:57:13.834Z 20 Acked 1019 Read 3 Done Done Sent false
64582024-02-07T18:57:13.834Z 21 Acked 1020 Read 5 Done Done Sent false
64592024-02-07T18:57:13.834Z 22 Acked 1021 Write 8 Done Done Sent false
64602024-02-07T18:57:13.834Z 23 Acked 1022 Write 5 Done Done Sent false
64612024-02-07T18:57:13.835Z 24 Acked 1023 Read 10 Done Done Sent false
64622024-02-07T18:57:13.835Z 25 Acked 1024 Write 6 Done Done Sent false
64632024-02-07T18:57:13.835Z 26 Acked 1025 Write 7 Done Done Sent false
64642024-02-07T18:57:13.835Z 27 Acked 1026 Write 6 Done Done Sent false
64652024-02-07T18:57:13.835Z 28 Acked 1027 Read 5 Done Done Sent false
64662024-02-07T18:57:13.835Z 29 Acked 1028 Write 8 Done Done Sent false
64672024-02-07T18:57:13.835Z 30 Acked 1029 Read 7 Done Done Sent false
64682024-02-07T18:57:13.835Z STATES DS:0 DS:1 DS:2 TOTAL
64692024-02-07T18:57:13.835Z New 0 0 0 0
64702024-02-07T18:57:13.835Z Sent 0 0 28 28
64712024-02-07T18:57:13.835Z Done 30 30 2 62
64722024-02-07T18:57:13.835Z Skipped 0 0 0 0
64732024-02-07T18:57:13.835Z Error 0 0 0 0
64742024-02-07T18:57:13.835ZLast Flush: 0 0 0
64752024-02-07T18:57:13.835ZDownstairs last five completed:
64762024-02-07T18:57:13.835ZUpstairs last five completed: 30 29 28 27 26
64772024-02-07T18:57:13.835Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:13.080504071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1367}
64782024-02-07T18:57:13.835ZWrote out file "/tmp/test_repair_verify.data"
64792024-02-07T18:57:13.835Z
64802024-02-07T18:57:13.845ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64812024-02-07T18:57:13.858ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64822024-02-07T18:57:13.858Z 0 000-019 57 57 57 94 94 94 T T F <---
64832024-02-07T18:57:13.859Z 1 020-039 57 57 57 94 94 94 F F F
64842024-02-07T18:57:13.859Z 2 040-059 57 57 57 94 94 94 F F F
64852024-02-07T18:57:13.859Z 3 060-079 51 51 51 87 87 87 T T F <---
64862024-02-07T18:57:13.859Z 4 080-099 57 57 57 94 94 94 T T F <---
64872024-02-07T18:57:13.859Z 5 100-119 57 57 57 94 94 94 T T F <---
64882024-02-07T18:57:13.859Z 6 120-139 57 57 57 94 94 94 T T F <---
64892024-02-07T18:57:13.859Z 7 140-159 56 56 56 92 92 92 F F F
64902024-02-07T18:57:13.859Z 8 160-179 51 51 51 87 87 87 F F F
64912024-02-07T18:57:13.859Z 9 180-199 57 57 57 94 94 94 F F F
64922024-02-07T18:57:13.859Z 10 200-219 57 57 57 94 94 94 F F F
64932024-02-07T18:57:13.859Z 11 220-239 52 52 52 88 88 88 T T F <---
64942024-02-07T18:57:13.862Z 12 240-259 49 49 49 85 85 85 T T T
64952024-02-07T18:57:13.862Z 13 260-279 49 49 49 85 85 85 F F F
64962024-02-07T18:57:13.862Z 14 280-299 56 56 56 92 92 92 F F F
64972024-02-07T18:57:13.862Z 15 300-319 57 57 57 94 94 94 T T F <---
64982024-02-07T18:57:13.862Z 16 320-339 49 49 49 85 85 85 T T F <---
64992024-02-07T18:57:13.862Z 17 340-359 57 57 57 94 94 94 F F F
65002024-02-07T18:57:13.862Z 18 360-379 55 55 55 91 91 91 F F F
65012024-02-07T18:57:13.862Z 19 380-399 53 53 53 89 89 89 F F F
65022024-02-07T18:57:13.862Z 20 400-419 57 57 57 94 94 94 T T F <---
65032024-02-07T18:57:13.862Z 21 420-439 51 51 51 87 87 87 T T T
65042024-02-07T18:57:13.863Z 22 440-459 57 57 57 94 94 94 T T F <---
65052024-02-07T18:57:13.863Z 23 460-479 53 53 53 89 89 89 T T F <---
65062024-02-07T18:57:13.863Z 24 480-499 46 46 46 83 83 83 F F F
65072024-02-07T18:57:13.863Z 25 500-519 44 44 44 80 80 80 T T F <---
65082024-02-07T18:57:13.863Z 26 520-539 55 55 55 91 91 91 T T F <---
65092024-02-07T18:57:13.863Z 27 540-559 53 53 53 89 89 89 F F F
65102024-02-07T18:57:13.863Z 28 560-579 49 49 49 85 85 85 F F F
65112024-02-07T18:57:13.863Z 29 580-599 57 57 57 94 94 94 F F F
65122024-02-07T18:57:13.863ZMax gen: 57, Max flush: 94
65132024-02-07T18:57:13.863ZError: Difference in extent metadata found!
65142024-02-07T18:57:13.863ZA Difference in extent metadata is expected here
65152024-02-07T18:57:13.863ZOn loop 29
65162024-02-07T18:57:13.863Z
65172024-02-07T18:57:13.866ZVerifying data now
65182024-02-07T18:57:19.613ZNo change after verify
65192024-02-07T18:57:19.613ZLoop: 29 Downstairs dump after verify (and repair):
65202024-02-07T18:57:19.628ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65212024-02-07T18:57:19.628Z 0 000-019 58 58 58 95 95 95 F F F
65222024-02-07T18:57:19.628Z 1 020-039 57 57 57 94 94 94 F F F
65232024-02-07T18:57:19.628Z 2 040-059 57 57 57 94 94 94 F F F
65242024-02-07T18:57:19.629Z 3 060-079 58 58 58 95 95 95 F F F
65252024-02-07T18:57:19.629Z 4 080-099 58 58 58 95 95 95 F F F
65262024-02-07T18:57:19.629Z 5 100-119 58 58 58 95 95 95 F F F
65272024-02-07T18:57:19.629Z 6 120-139 58 58 58 95 95 95 F F F
65282024-02-07T18:57:19.629Z 7 140-159 56 56 56 92 92 92 F F F
65292024-02-07T18:57:19.629Z 8 160-179 51 51 51 87 87 87 F F F
65302024-02-07T18:57:19.629Z 9 180-199 57 57 57 94 94 94 F F F
65312024-02-07T18:57:19.629Z 10 200-219 57 57 57 94 94 94 F F F
65322024-02-07T18:57:19.629Z 11 220-239 58 58 58 95 95 95 F F F
65332024-02-07T18:57:19.629Z 12 240-259 58 58 58 95 95 95 F F F
65342024-02-07T18:57:19.629Z 13 260-279 49 49 49 85 85 85 F F F
65352024-02-07T18:57:19.629Z 14 280-299 56 56 56 92 92 92 F F F
65362024-02-07T18:57:19.629Z 15 300-319 58 58 58 95 95 95 F F F
65372024-02-07T18:57:19.629Z 16 320-339 58 58 58 95 95 95 F F F
65382024-02-07T18:57:19.632Z 17 340-359 57 57 57 94 94 94 F F F
65392024-02-07T18:57:19.632Z 18 360-379 55 55 55 91 91 91 F F F
65402024-02-07T18:57:19.632Z 19 380-399 53 53 53 89 89 89 F F F
65412024-02-07T18:57:19.632Z 20 400-419 58 58 58 95 95 95 F F F
65422024-02-07T18:57:19.632Z 21 420-439 58 58 58 95 95 95 F F F
65432024-02-07T18:57:19.633Z 22 440-459 58 58 58 95 95 95 F F F
65442024-02-07T18:57:19.633Z 23 460-479 58 58 58 95 95 95 F F F
65452024-02-07T18:57:19.633Z 24 480-499 46 46 46 83 83 83 F F F
65462024-02-07T18:57:19.633Z 25 500-519 58 58 58 95 95 95 F F F
65472024-02-07T18:57:19.633Z 26 520-539 58 58 58 95 95 95 F F F
65482024-02-07T18:57:19.633Z 27 540-559 53 53 53 89 89 89 F F F
65492024-02-07T18:57:19.633Z 28 560-579 49 49 49 85 85 85 F F F
65502024-02-07T18:57:19.633Z 29 580-599 57 57 57 94 94 94 F F F
65512024-02-07T18:57:19.633ZMax gen: 58, Max flush: 95
65522024-02-07T18:57:19.633Z
65532024-02-07T18:57:19.633ZBegin loop 30
65542024-02-07T18:57:19.633ZDownstairs to restart: 1
65552024-02-07T18:57:19.651Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901288266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375}
65562024-02-07T18:57:19.651Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
65572024-02-07T18:57:19.651Z,"time":"2024-02-07T18:57:18.90162676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375}
65582024-02-07T18:57:19.651Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901646917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375}
65592024-02-07T18:57:19.656Z{"msg":"Crucible 23effbc9-f1e5-47ba-9ef9-3ded88fd458b has session id: 1992c320-4fc6-464a-97ef-fcbd5eb01045","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901668114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65602024-02-07T18:57:19.656Z{"msg":"Upstairs opts: Upstairs UUID: 23effbc9-f1e5-47ba-9ef9-3ded88fd458b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901715868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65612024-02-07T18:57:19.656Z{"msg":"Crucible stats registered with UUID: 23effbc9-f1e5-47ba-9ef9-3ded88fd458b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901754113Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65622024-02-07T18:57:19.656Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901771231Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375}
65632024-02-07T18:57:19.656Z{"msg":"23effbc9-f1e5-47ba-9ef9-3ded88fd458b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901947996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65642024-02-07T18:57:19.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.901979642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65652024-02-07T18:57:19.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.9019966Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65662024-02-07T18:57:19.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902016437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65672024-02-07T18:57:19.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902047243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65682024-02-07T18:57:19.657Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.90207259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65692024-02-07T18:57:19.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902094377Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65702024-02-07T18:57:19.657Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902117244Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65712024-02-07T18:57:19.657Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.90214341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65722024-02-07T18:57:19.657Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902185564Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65732024-02-07T18:57:19.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902320256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65742024-02-07T18:57:19.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902404075Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65752024-02-07T18:57:19.657Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:18.90258715Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65762024-02-07T18:57:19.657Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:18.902638833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65772024-02-07T18:57:19.657Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:18.90273382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65782024-02-07T18:57:19.657Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902770665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65792024-02-07T18:57:19.657Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902802641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65802024-02-07T18:57:19.657Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.902834987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65812024-02-07T18:57:19.657Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.903167372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65822024-02-07T18:57:19.657Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.903213835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65832024-02-07T18:57:19.657Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.903259069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65842024-02-07T18:57:19.657Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.903301013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65852024-02-07T18:57:19.657Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.903714908Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65862024-02-07T18:57:19.657Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.903849159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65872024-02-07T18:57:19.657Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.904266513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65882024-02-07T18:57:19.657Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.904306637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65892024-02-07T18:57:19.657Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.904351131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65902024-02-07T18:57:19.657Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.904390596Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65912024-02-07T18:57:19.658Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.904416783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65922024-02-07T18:57:19.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:18.904431271Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65932024-02-07T18:57:20.654Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.904267972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65942024-02-07T18:57:20.659Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.904491162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65952024-02-07T18:57:20.659Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.905293863Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65962024-02-07T18:57:20.659Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.905368403Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65972024-02-07T18:57:20.659Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.905933227Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65982024-02-07T18:57:20.659Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906583449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
65992024-02-07T18:57:20.659Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906629252Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66002024-02-07T18:57:20.659Z{"msg":"[0]R flush_numbers[0..12]: [95, 94, 94, 95, 95, 95, 95, 92, 87, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906656339Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66012024-02-07T18:57:20.659Z{"msg":"[0]R generation[0..12]: [58, 57, 57, 58, 58, 58, 58, 56, 51, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906691854Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66022024-02-07T18:57:20.659Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906728669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66032024-02-07T18:57:20.659Z{"msg":"[1]R flush_numbers[0..12]: [95, 94, 94, 95, 95, 95, 95, 92, 87, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906766014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66042024-02-07T18:57:20.659Z{"msg":"[1]R generation[0..12]: [58, 57, 57, 58, 58, 58, 58, 56, 51, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906799909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66052024-02-07T18:57:20.659ZWait for a query_work_queue command to finish before sending IO
66062024-02-07T18:57:20.659Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906845743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66072024-02-07T18:57:20.659Z{"msg":"[2]R flush_numbers[0..12]: [95, 94, 94, 95, 95, 95, 95, 92, 87, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906881078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66082024-02-07T18:57:20.659Z{"msg":"[2]R generation[0..12]: [58, 57, 57, 58, 58, 58, 58, 56, 51, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906920853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66092024-02-07T18:57:20.659Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906960588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66102024-02-07T18:57:20.659Z{"msg":"Max found gen is 59","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.906986814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66112024-02-07T18:57:20.659Z{"msg":"Generation requested: 60 >= found:59","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907013711Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66122024-02-07T18:57:20.659Z{"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907053175Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66132024-02-07T18:57:20.660Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.90708983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66142024-02-07T18:57:20.660Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907117976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66152024-02-07T18:57:20.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907143473Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66162024-02-07T18:57:20.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.90716637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66172024-02-07T18:57:20.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907192946Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66182024-02-07T18:57:20.660Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907230101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66192024-02-07T18:57:20.660Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907256308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66202024-02-07T18:57:20.660Z{"msg":"23effbc9-f1e5-47ba-9ef9-3ded88fd458b is now active with session: 1992c320-4fc6-464a-97ef-fcbd5eb01045","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907291783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66212024-02-07T18:57:20.660Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
66222024-02-07T18:57:20.660Z2024-02-07T18:57:19.907326008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375,"session_id":"1992c320-4fc6-464a-97ef-fcbd5eb01045"}
66232024-02-07T18:57:20.660Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:19.907356484Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375}
66242024-02-07T18:57:20.660ZLoading write count information from file "/tmp/test_repair_verify.data"
66252024-02-07T18:57:20.660ZRun Repair workload
66262024-02-07T18:57:20.660Z01/30 Write block 44 len 16384 data: 5 5 5 6
66272024-02-07T18:57:20.670Z02/30 Read block 337 len 4096
66282024-02-07T18:57:20.670Z03/30 Read block 192 len 32768
66292024-02-07T18:57:20.670Z04/30 Write block 136 len 32768 data: 2 3 3 4 3 6 6 6
66302024-02-07T18:57:20.671Z05/30 Write block 585 len 40960 data: 5 6 5 6 6 6 7 7 6 6
66312024-02-07T18:57:20.671Z06/30 Read block 306 len 20480
66322024-02-07T18:57:20.683Z07/30 Read block 370 len 8192
66332024-02-07T18:57:20.683Z08/30 Read block 53 len 36864
66342024-02-07T18:57:20.683Z09/30 Write block 393 len 8192 data: 8 7
66352024-02-07T18:57:20.683Z10/30 Read block 66 len 32768
66362024-02-07T18:57:20.684Z11/30 Write block 578 len 8192 data: 4 4
66372024-02-07T18:57:20.684Z12/30 Write block 252 len 12288 data: 4 3 3
66382024-02-07T18:57:20.684Z13/30 Read block 518 len 28672
66392024-02-07T18:57:20.684Z14/30 Write block 520 len 40960 data: 4 2 2 2 5 4 5 6 6 6
66402024-02-07T18:57:20.684Z15/30 Read block 524 len 32768
66412024-02-07T18:57:20.684Z16/30 Read block 491 len 28672
66422024-02-07T18:57:20.684Z17/30 Read block 401 len 36864
66432024-02-07T18:57:20.684Z18/30 Read block 81 len 4096
66442024-02-07T18:57:20.684Z19/30 Read block 557 len 24576
66452024-02-07T18:57:20.685Z20/30 Read block 388 len 4096
66462024-02-07T18:57:20.685Z21/30 Read block 499 len 28672
66472024-02-07T18:57:20.685Z22/30 Read block 195 len 4096
66482024-02-07T18:57:20.685Z23/30 Read block 341 len 16384
66492024-02-07T18:57:20.700Z24/30 Read block 30 len 40960
66502024-02-07T18:57:20.700Z25/30 Flush
66512024-02-07T18:57:20.700Z26/30 Write block 294 len 36864 data: 4 5 4 6 5 3 3 2 3
66522024-02-07T18:57:20.700Z27/30 Read block 142 len 16384
66532024-02-07T18:57:20.700Z28/30 Write block 7 len 20480 data: 6 5 6 5 5
66542024-02-07T18:57:20.700Z29/30 Read block 355 len 28672
66552024-02-07T18:57:20.700Z30/30 Read block 319 len 12288
66562024-02-07T18:57:20.700Z----------------------------------------------------------------
66572024-02-07T18:57:20.700Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
66582024-02-07T18:57:20.700ZStates: Active Active Active
66592024-02-07T18:57:20.700ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
66602024-02-07T18:57:20.700Z 1 Acked 1000 Write 4 Done Done Done false
66612024-02-07T18:57:20.700Z 2 Acked 1001 Read 1 Done Done Done false
66622024-02-07T18:57:20.700Z 3 Acked 1002 Read 8 Done Done Done false
66632024-02-07T18:57:20.700Z 4 Acked 1003 Write 8 Done Done Done false
66642024-02-07T18:57:20.700Z 5 Acked 1004 Write 10 Done Done Done false
66652024-02-07T18:57:20.700Z 6 Acked 1005 Read 5 Done Sent Done false
66662024-02-07T18:57:20.700Z 7 Acked 1006 Read 2 Done Sent Done false
66672024-02-07T18:57:20.700Z 8 Acked 1007 Read 9 Done Sent Done false
66682024-02-07T18:57:20.700Z 9 Acked 1008 Write 2 Done Sent Done false
66692024-02-07T18:57:20.701Z 10 Acked 1009 Read 8 Done Sent Done false
66702024-02-07T18:57:20.701Z 11 Acked 1010 Write 2 Done Sent Done false
66712024-02-07T18:57:20.701Z 12 Acked 1011 Write 3 Done Sent Done false
66722024-02-07T18:57:20.701Z 13 Acked 1012 Read 7 Done Sent Done false
66732024-02-07T18:57:20.701Z 14 Acked 1013 Write 10 Done Sent Done false
66742024-02-07T18:57:20.701Z 15 Acked 1014 Read 8 Done Sent Done false
66752024-02-07T18:57:20.701Z 16 Acked 1015 Read 7 Done Sent Done false
66762024-02-07T18:57:20.701Z 17 Acked 1016 Read 9 Done Sent Done false
66772024-02-07T18:57:20.701Z 18 Acked 1017 Read 1 Done Sent Done false
66782024-02-07T18:57:20.701Z 19 Acked 1018 Read 6 Done Sent Done false
66792024-02-07T18:57:20.701Z 20 Acked 1019 Read 1 Done Sent Done false
66802024-02-07T18:57:20.701Z 21 Acked 1020 Read 7 Done Sent Done false
66812024-02-07T18:57:20.701Z 22 Acked 1021 Read 1 Done Sent Done false
66822024-02-07T18:57:20.701Z 23 Acked 1022 Read 4 Done Sent Done false
66832024-02-07T18:57:20.701Z 24 Acked 1023 Read 10 Done Sent Done false
66842024-02-07T18:57:20.701Z 25 Acked 1024 Flush 0 Done Sent Done false
66852024-02-07T18:57:20.701Z 26 Acked 1025 Write 9 Done Sent Done false
66862024-02-07T18:57:20.701Z 27 Acked 1026 Read 4 Done Sent Done false
66872024-02-07T18:57:20.701Z 28 Acked 1027 Write 5 Done Sent Done false
66882024-02-07T18:57:20.701Z 29 Acked 1028 Read 7 Done Sent Done false
66892024-02-07T18:57:20.701Z 30 Acked 1029 Read 3 Done Sent Done false
66902024-02-07T18:57:20.701Z STATES DS:0 DS:1 DS:2 TOTAL
66912024-02-07T18:57:20.702Z New 0 0 0 0
66922024-02-07T18:57:20.702Z Sent 0 25 0 25
66932024-02-07T18:57:20.702Z Done 30 5 30 65
66942024-02-07T18:57:20.702Z Skipped 0 0 0 0
66952024-02-07T18:57:20.702Z Error 0 0 0 0
66962024-02-07T18:57:20.702ZLast Flush: 1024 0 1024
66972024-02-07T18:57:20.702ZDownstairs last five completed:
66982024-02-07T18:57:20.702ZUpstairs last five completed: 30 29 28 27 26
66992024-02-07T18:57:20.702Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:19.936779265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1375}
67002024-02-07T18:57:20.702ZWrote out file "/tmp/test_repair_verify.data"
67012024-02-07T18:57:20.702Z
67022024-02-07T18:57:20.706ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67032024-02-07T18:57:20.710ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67042024-02-07T18:57:20.711Z 0 000-019 58 58 58 95 95 95 T F T <---
67052024-02-07T18:57:20.711Z 1 020-039 57 57 57 94 94 94 F F F
67062024-02-07T18:57:20.711Z 2 040-059 60 57 60 96 94 96 F T F <---
67072024-02-07T18:57:20.711Z 3 060-079 58 58 58 95 95 95 F F F
67082024-02-07T18:57:20.711Z 4 080-099 58 58 58 95 95 95 F F F
67092024-02-07T18:57:20.714Z 5 100-119 58 58 58 95 95 95 F F F
67102024-02-07T18:57:20.714Z 6 120-139 60 58 60 96 95 96 F T F <---
67112024-02-07T18:57:20.714Z 7 140-159 60 56 60 96 92 96 F T F <---
67122024-02-07T18:57:20.714Z 8 160-179 51 51 51 87 87 87 F F F
67132024-02-07T18:57:20.714Z 9 180-199 57 57 57 94 94 94 F F F
67142024-02-07T18:57:20.714Z 10 200-219 57 57 57 94 94 94 F F F
67152024-02-07T18:57:20.714Z 11 220-239 58 58 58 95 95 95 F F F
67162024-02-07T18:57:20.714Z 12 240-259 60 58 60 96 95 96 F F F <---
67172024-02-07T18:57:20.714Z 13 260-279 49 49 49 85 85 85 F F F
67182024-02-07T18:57:20.714Z 14 280-299 56 56 56 92 92 92 T F T <---
67192024-02-07T18:57:20.714Z 15 300-319 58 58 58 95 95 95 T F T <---
67202024-02-07T18:57:20.714Z 16 320-339 58 58 58 95 95 95 F F F
67212024-02-07T18:57:20.715Z 17 340-359 57 57 57 94 94 94 F F F
67222024-02-07T18:57:20.715Z 18 360-379 55 55 55 91 91 91 F F F
67232024-02-07T18:57:20.715Z 19 380-399 60 53 60 96 89 96 F F F <---
67242024-02-07T18:57:20.715Z 20 400-419 58 58 58 95 95 95 F F F
67252024-02-07T18:57:20.715Z 21 420-439 58 58 58 95 95 95 F F F
67262024-02-07T18:57:20.715Z 22 440-459 58 58 58 95 95 95 F F F
67272024-02-07T18:57:20.715Z 23 460-479 58 58 58 95 95 95 F F F
67282024-02-07T18:57:20.715Z 24 480-499 46 46 46 83 83 83 F F F
67292024-02-07T18:57:20.715Z 25 500-519 58 58 58 95 95 95 F F F
67302024-02-07T18:57:20.715Z 26 520-539 60 58 60 96 95 96 F F F <---
67312024-02-07T18:57:20.715Z 27 540-559 53 53 53 89 89 89 F F F
67322024-02-07T18:57:20.715Z 28 560-579 60 49 60 96 85 96 F F F <---
67332024-02-07T18:57:20.715Z 29 580-599 60 57 60 96 94 96 F T F <---
67342024-02-07T18:57:20.715ZMax gen: 60, Max flush: 96
67352024-02-07T18:57:20.715ZError: Difference in extent metadata found!
67362024-02-07T18:57:20.715ZA Difference in extent metadata is expected here
67372024-02-07T18:57:20.715ZOn loop 30
67382024-02-07T18:57:20.715Z
67392024-02-07T18:57:20.719ZVerifying data now
67402024-02-07T18:57:29.393ZNo change after verify
67412024-02-07T18:57:29.393ZLoop: 30 Downstairs dump after verify (and repair):
67422024-02-07T18:57:29.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67432024-02-07T18:57:29.458Z 0 000-019 61 61 61 97 97 97 F F F
67442024-02-07T18:57:29.458Z 1 020-039 57 57 57 94 94 94 F F F
67452024-02-07T18:57:29.458Z 2 040-059 60 60 60 96 96 96 F F F
67462024-02-07T18:57:29.458Z 3 060-079 58 58 58 95 95 95 F F F
67472024-02-07T18:57:29.458Z 4 080-099 58 58 58 95 95 95 F F F
67482024-02-07T18:57:29.458Z 5 100-119 58 58 58 95 95 95 F F F
67492024-02-07T18:57:29.458Z 6 120-139 60 60 60 96 96 96 F F F
67502024-02-07T18:57:29.458Z 7 140-159 60 60 60 96 96 96 F F F
67512024-02-07T18:57:29.458Z 8 160-179 51 51 51 87 87 87 F F F
67522024-02-07T18:57:29.458Z 9 180-199 57 57 57 94 94 94 F F F
67532024-02-07T18:57:29.458Z 10 200-219 57 57 57 94 94 94 F F F
67542024-02-07T18:57:29.458Z 11 220-239 58 58 58 95 95 95 F F F
67552024-02-07T18:57:29.458Z 12 240-259 60 60 60 96 96 96 F F F
67562024-02-07T18:57:29.458Z 13 260-279 49 49 49 85 85 85 F F F
67572024-02-07T18:57:29.458Z 14 280-299 61 61 61 97 97 97 F F F
67582024-02-07T18:57:29.458Z 15 300-319 61 61 61 97 97 97 F F F
67592024-02-07T18:57:29.458Z 16 320-339 58 58 58 95 95 95 F F F
67602024-02-07T18:57:29.458Z 17 340-359 57 57 57 94 94 94 F F F
67612024-02-07T18:57:29.458Z 18 360-379 55 55 55 91 91 91 F F F
67622024-02-07T18:57:29.459Z 19 380-399 60 60 60 96 96 96 F F F
67632024-02-07T18:57:29.459Z 20 400-419 58 58 58 95 95 95 F F F
67642024-02-07T18:57:29.459Z 21 420-439 58 58 58 95 95 95 F F F
67652024-02-07T18:57:29.459Z 22 440-459 58 58 58 95 95 95 F F F
67662024-02-07T18:57:29.459Z 23 460-479 58 58 58 95 95 95 F F F
67672024-02-07T18:57:29.459Z 24 480-499 46 46 46 83 83 83 F F F
67682024-02-07T18:57:29.459Z 25 500-519 58 58 58 95 95 95 F F F
67692024-02-07T18:57:29.459Z 26 520-539 60 60 60 96 96 96 F F F
67702024-02-07T18:57:29.459Z 27 540-559 53 53 53 89 89 89 F F F
67712024-02-07T18:57:29.459Z 28 560-579 60 60 60 96 96 96 F F F
67722024-02-07T18:57:29.459Z 29 580-599 60 60 60 96 96 96 F F F
67732024-02-07T18:57:29.459ZMax gen: 61, Max flush: 97
67742024-02-07T18:57:29.459Z
67752024-02-07T18:57:29.459ZBegin loop 31
67762024-02-07T18:57:29.459ZDownstairs to restart: 2
67772024-02-07T18:57:29.459Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.561735606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383}
67782024-02-07T18:57:29.459Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562017788Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383}
67792024-02-07T18:57:29.459Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
67802024-02-07T18:57:29.459Z,"time":"2024-02-07T18:57:25.562057023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383}
67812024-02-07T18:57:29.459Z{"msg":"Crucible e5723aea-1d3a-49f6-b8cf-d929668200e8 has session id: 3a3837b5-707c-43cc-b242-eb7a5c406987","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562085379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67822024-02-07T18:57:29.459Z{"msg":"Upstairs opts: Upstairs UUID: e5723aea-1d3a-49f6-b8cf-d929668200e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562141101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67832024-02-07T18:57:29.460Z{"msg":"Crucible stats registered with UUID: e5723aea-1d3a-49f6-b8cf-d929668200e8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562175736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67842024-02-07T18:57:29.460Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562199353Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383}
67852024-02-07T18:57:29.460Z{"msg":"e5723aea-1d3a-49f6-b8cf-d929668200e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.5622211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67862024-02-07T18:57:29.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562234419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67872024-02-07T18:57:29.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562249257Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67882024-02-07T18:57:29.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562270334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67892024-02-07T18:57:29.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562284562Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67902024-02-07T18:57:29.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.56229857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67912024-02-07T18:57:29.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562312038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67922024-02-07T18:57:29.460Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562364121Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67932024-02-07T18:57:29.460Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562393657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67942024-02-07T18:57:29.460Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562425333Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67952024-02-07T18:57:29.460Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562640524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67962024-02-07T18:57:29.460Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562713294Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67972024-02-07T18:57:29.460Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:25.562757238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67982024-02-07T18:57:29.460Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:25.562804442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
67992024-02-07T18:57:29.460Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:25.562838777Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68002024-02-07T18:57:29.460Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562853885Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68012024-02-07T18:57:29.460Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562869453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68022024-02-07T18:57:29.460Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.562899599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68032024-02-07T18:57:29.460Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.563418818Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68042024-02-07T18:57:29.460Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.563454473Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68052024-02-07T18:57:29.460Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.563470331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68062024-02-07T18:57:29.460Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.563484169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68072024-02-07T18:57:29.461Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564030905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68082024-02-07T18:57:29.461Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564174116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68092024-02-07T18:57:29.461Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564568412Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68102024-02-07T18:57:29.461Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564641682Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68112024-02-07T18:57:29.461Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564674858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68122024-02-07T18:57:29.461Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564705334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68132024-02-07T18:57:29.461Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564720982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68142024-02-07T18:57:29.461Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:25.564742469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68152024-02-07T18:57:29.461Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.564527317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68162024-02-07T18:57:29.461Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.564601857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68172024-02-07T18:57:29.461Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.565449252Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68182024-02-07T18:57:29.461Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.565493396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68192024-02-07T18:57:29.461Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566009326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68202024-02-07T18:57:29.461Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566613615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68212024-02-07T18:57:29.461Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566664888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68222024-02-07T18:57:29.461Z{"msg":"[0]R flush_numbers[0..12]: [97, 94, 96, 95, 95, 95, 96, 96, 87, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566699073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68232024-02-07T18:57:29.461Z{"msg":"[0]R generation[0..12]: [61, 57, 60, 58, 58, 58, 60, 60, 51, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.56671825Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68242024-02-07T18:57:29.461Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566744977Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68252024-02-07T18:57:29.461Z{"msg":"[1]R flush_numbers[0..12]: [97, 94, 96, 95, 95, 95, 96, 96, 87, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566764624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68262024-02-07T18:57:29.461Z{"msg":"[1]R generation[0..12]: [61, 57, 60, 58, 58, 58, 60, 60, 51, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566789881Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68272024-02-07T18:57:29.462Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566809218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68282024-02-07T18:57:29.462Z{"msg":"[2]R flush_numbers[0..12]: [97, 94, 96, 95, 95, 95, 96, 96, 87, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566826566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68292024-02-07T18:57:29.462ZWait for a query_work_queue command to finish before sending IO
68302024-02-07T18:57:29.462Z{"msg":"[2]R generation[0..12]: [61, 57, 60, 58, 58, 58, 60, 60, 51, 57, 57, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566860181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68312024-02-07T18:57:29.462Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566886348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68322024-02-07T18:57:29.462Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566904405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68332024-02-07T18:57:29.462Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566950069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68342024-02-07T18:57:29.462Z{"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566967217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68352024-02-07T18:57:29.462Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.566984685Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68362024-02-07T18:57:29.462Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567010371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68372024-02-07T18:57:29.462Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567026469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68382024-02-07T18:57:29.462Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567057385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68392024-02-07T18:57:29.462Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567084631Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68402024-02-07T18:57:29.462Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567120616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68412024-02-07T18:57:29.462Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567155941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68422024-02-07T18:57:29.484Z{"msg":"e5723aea-1d3a-49f6-b8cf-d929668200e8 is now active with session: 3a3837b5-707c-43cc-b242-eb7a5c406987","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567191686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68432024-02-07T18:57:29.484Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567209214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383,"session_id":"3a3837b5-707c-43cc-b242-eb7a5c406987"}
68442024-02-07T18:57:29.484Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:26.567245269Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383}
68452024-02-07T18:57:29.484ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
68462024-02-07T18:57:29.485ZLoading write count information from file "/tmp/test_repair_verify.data"
68472024-02-07T18:57:29.485ZRun Repair workload
68482024-02-07T18:57:29.485Z01/30 Write block 336 len 32768 data: 6 5 7 9 7 5 5 5
68492024-02-07T18:57:29.485Z02/30 Read block 204 len 24576
68502024-02-07T18:57:29.485Z03/30 Write block 425 len 20480 data: 8 7 6 8 7
68512024-02-07T18:57:29.485Z04/30 Read block 318 len 4096
68522024-02-07T18:57:29.485Z05/30 Read block 80 len 28672
68532024-02-07T18:57:29.485Z06/30 Read block 574 len 24576
68542024-02-07T18:57:29.485Z07/30 Flush
68552024-02-07T18:57:29.485Z08/30 Write block 493 len 16384 data: 6 8 5 5
68562024-02-07T18:57:29.485Z09/30 Write block 559 len 8192 data: 5 4
68572024-02-07T18:57:29.485Z10/30 Read block 138 len 40960
68582024-02-07T18:57:29.486Z11/30 Write block 512 len 20480 data: 8 7 7 6 5
68592024-02-07T18:57:29.486Z12/30 Read block 297 len 4096
68602024-02-07T18:57:29.486Z13/30 Read block 76 len 40960
68612024-02-07T18:57:29.486Z14/30 Write block 533 len 40960 data: 6 6 8 9 10 8 8 8 8 10
68622024-02-07T18:57:29.486Z15/30 Write block 238 len 28672 data: 8 7 8 8 8 9 8
68632024-02-07T18:57:29.486Z16/30 Read block 86 len 20480
68642024-02-07T18:57:29.486Z17/30 Write block 286 len 36864 data: 8 7 7 7 8 6 4 5 5
68652024-02-07T18:57:29.486Z18/30 Flush
68662024-02-07T18:57:29.486Z19/30 Write block 285 len 16384 data: 6 9 8 8
68672024-02-07T18:57:29.486Z20/30 Read block 311 len 16384
68682024-02-07T18:57:29.486Z21/30 Write block 240 len 36864 data: 9 9 9 10 9 6 6 6 5
68692024-02-07T18:57:29.486Z22/30 Read block 278 len 16384
68702024-02-07T18:57:29.486Z23/30 Write block 444 len 20480 data: 8 7 7 5 5
68712024-02-07T18:57:29.487Z24/30 Read block 570 len 40960
68722024-02-07T18:57:29.487Z25/30 Read block 408 len 16384
68732024-02-07T18:57:29.487Z26/30 Write block 359 len 20480 data: 5 7 8 9 8
68742024-02-07T18:57:29.487Z27/30 Write block 413 len 16384 data: 8 7 7 5
68752024-02-07T18:57:29.487Z28/30 Read block 429 len 36864
68762024-02-07T18:57:29.487Z29/30 Read block 253 len 36864
68772024-02-07T18:57:29.487Z30/30 Write block 309 len 40960 data: 5 5 5 3 4 5 8 10 8 8
68782024-02-07T18:57:29.487Z----------------------------------------------------------------
68792024-02-07T18:57:29.487Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
68802024-02-07T18:57:29.487ZStates: Active Active Active
68812024-02-07T18:57:29.487ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
68822024-02-07T18:57:29.487Z 1 Acked 1000 Write 8 Done Done Done false
68832024-02-07T18:57:29.488Z 2 Acked 1001 Read 6 Done Done Done false
68842024-02-07T18:57:29.488Z 3 Acked 1002 Write 5 Done Done Done false
68852024-02-07T18:57:29.488Z 4 Acked 1003 Read 1 Done Done Done false
68862024-02-07T18:57:29.488Z 5 Acked 1004 Read 7 Done Done Done false
68872024-02-07T18:57:29.488Z 6 Acked 1005 Read 6 Done Done Done false
68882024-02-07T18:57:29.488Z 7 Acked 1006 Flush 0 Done Done Sent false
68892024-02-07T18:57:29.488Z 8 Acked 1007 Write 4 Done Done Sent false
68902024-02-07T18:57:29.488Z 9 Acked 1008 Write 2 Done Done Sent false
68912024-02-07T18:57:29.488Z 10 Acked 1009 Read 10 Done Done Sent false
68922024-02-07T18:57:29.488Z 11 Acked 1010 Write 5 Done Done Sent false
68932024-02-07T18:57:29.488Z 12 Acked 1011 Read 1 Done Done Sent false
68942024-02-07T18:57:29.489Z 13 Acked 1012 Read 10 Done Done Sent false
68952024-02-07T18:57:29.489Z 14 Acked 1013 Write 10 Done Done Sent false
68962024-02-07T18:57:29.489Z 15 Acked 1014 Write 7 Done Done Sent false
68972024-02-07T18:57:29.489Z 16 Acked 1015 Read 5 Done Done Sent false
68982024-02-07T18:57:29.489Z 17 Acked 1016 Write 9 Done Done Sent false
68992024-02-07T18:57:29.489Z 18 Acked 1017 Flush 0 Done Done Sent false
69002024-02-07T18:57:29.489Z 19 Acked 1018 Write 4 Done Done Sent false
69012024-02-07T18:57:29.489Z 20 Acked 1019 Read 4 Done Done Sent false
69022024-02-07T18:57:29.489Z 21 Acked 1020 Write 9 Done Done Sent false
69032024-02-07T18:57:29.489Z 22 Acked 1021 Read 4 Done Done Sent false
69042024-02-07T18:57:29.489Z 23 Acked 1022 Write 5 Done Done Sent false
69052024-02-07T18:57:29.489Z 24 Acked 1023 Read 10 Done Done Sent false
69062024-02-07T18:57:29.490Z 25 Acked 1024 Read 4 Done Done Sent false
69072024-02-07T18:57:29.490Z 26 Acked 1025 Write 5 Done Done Sent false
69082024-02-07T18:57:29.490Z 27 Acked 1026 Write 4 Done Done Sent false
69092024-02-07T18:57:29.490Z 28 Acked 1027 Read 9 Done Done Sent false
69102024-02-07T18:57:29.490Z 29 Acked 1028 Read 9 Done Done Sent false
69112024-02-07T18:57:29.490Z 30 Acked 1029 Write 10 Sent Sent Sent false
69122024-02-07T18:57:29.490Z STATES DS:0 DS:1 DS:2 TOTAL
69132024-02-07T18:57:29.490Z New 0 0 0 0
69142024-02-07T18:57:29.490Z Sent 1 1 24 26
69152024-02-07T18:57:29.490Z Done 29 29 6 64
69162024-02-07T18:57:29.490Z Skipped 0 0 0 0
69172024-02-07T18:57:29.491Z Error 0 0 0 0
69182024-02-07T18:57:29.491ZLast Flush: 1017 1017 0
69192024-02-07T18:57:29.491ZDownstairs last five completed:
69202024-02-07T18:57:29.491ZUpstairs last five completed: 30 29 28 27 26
69212024-02-07T18:57:29.491Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:26.59910628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1383}
69222024-02-07T18:57:29.491ZWrote out file "/tmp/test_repair_verify.data"
69232024-02-07T18:57:29.491Z
69242024-02-07T18:57:29.491ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
69252024-02-07T18:57:29.491ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69262024-02-07T18:57:29.491Z 0 000-019 61 61 61 97 97 97 F F F
69272024-02-07T18:57:29.491Z 1 020-039 57 57 57 94 94 94 F F F
69282024-02-07T18:57:29.491Z 2 040-059 60 60 60 96 96 96 F F F
69292024-02-07T18:57:29.492Z 3 060-079 58 58 58 95 95 95 F F F
69302024-02-07T18:57:29.492Z 4 080-099 58 58 58 95 95 95 F F F
69312024-02-07T18:57:29.492Z 5 100-119 58 58 58 95 95 95 F F F
69322024-02-07T18:57:29.492Z 6 120-139 60 60 60 96 96 96 F F F
69332024-02-07T18:57:29.492Z 7 140-159 60 60 60 96 96 96 F F F
69342024-02-07T18:57:29.492Z 8 160-179 51 51 51 87 87 87 F F F
69352024-02-07T18:57:29.492Z 9 180-199 57 57 57 94 94 94 F F F
69362024-02-07T18:57:29.492Z 10 200-219 57 57 57 94 94 94 F F F
69372024-02-07T18:57:29.492Z 11 220-239 62 62 58 99 99 95 F F F <---
69382024-02-07T18:57:29.492Z 12 240-259 62 62 60 99 99 96 T T F <---
69392024-02-07T18:57:29.492Z 13 260-279 49 49 49 85 85 85 F F F
69402024-02-07T18:57:29.493Z 14 280-299 62 62 61 99 99 97 T T F <---
69412024-02-07T18:57:29.493Z 15 300-319 61 61 61 97 97 97 T T F <---
69422024-02-07T18:57:29.506Z 16 320-339 62 62 58 98 98 95 F F T <---
69432024-02-07T18:57:29.506Z 17 340-359 62 62 57 98 98 94 T T T <---
69442024-02-07T18:57:29.506Z 18 360-379 55 55 55 91 91 91 T T F <---
69452024-02-07T18:57:29.506Z 19 380-399 60 60 60 96 96 96 F F F
69462024-02-07T18:57:29.506Z 20 400-419 58 58 58 95 95 95 T T F <---
69472024-02-07T18:57:29.506Z 21 420-439 62 62 58 98 98 95 F F T <---
69482024-02-07T18:57:29.507Z 22 440-459 58 58 58 95 95 95 T T F <---
69492024-02-07T18:57:29.507Z 23 460-479 58 58 58 95 95 95 F F F
69502024-02-07T18:57:29.507Z 24 480-499 62 62 46 99 99 83 F F F <---
69512024-02-07T18:57:29.508Z 25 500-519 62 62 58 99 99 95 F F F <---
69522024-02-07T18:57:29.508Z 26 520-539 62 62 60 99 99 96 F F F <---
69532024-02-07T18:57:29.508Z 27 540-559 62 62 53 99 99 89 F F F <---
69542024-02-07T18:57:29.508Z 28 560-579 62 62 60 99 99 96 F F F <---
69552024-02-07T18:57:29.508Z 29 580-599 60 60 60 96 96 96 F F F
69562024-02-07T18:57:29.509ZMax gen: 62, Max flush: 99
69572024-02-07T18:57:29.509ZError: Difference in extent metadata found!
69582024-02-07T18:57:29.509ZA Difference in extent metadata is expected here
69592024-02-07T18:57:29.509ZOn loop 31
69602024-02-07T18:57:29.509Z
69612024-02-07T18:57:29.509ZVerifying data now
69622024-02-07T18:57:33.096ZNo change after verify
69632024-02-07T18:57:33.096ZLoop: 31 Downstairs dump after verify (and repair):
69642024-02-07T18:57:33.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69652024-02-07T18:57:33.110Z 0 000-019 61 61 61 97 97 97 F F F
69662024-02-07T18:57:33.110Z 1 020-039 57 57 57 94 94 94 F F F
69672024-02-07T18:57:33.110Z 2 040-059 60 60 60 96 96 96 F F F
69682024-02-07T18:57:33.110Z 3 060-079 58 58 58 95 95 95 F F F
69692024-02-07T18:57:33.111Z 4 080-099 58 58 58 95 95 95 F F F
69702024-02-07T18:57:33.111Z 5 100-119 58 58 58 95 95 95 F F F
69712024-02-07T18:57:33.111Z 6 120-139 60 60 60 96 96 96 F F F
69722024-02-07T18:57:33.111Z 7 140-159 60 60 60 96 96 96 F F F
69732024-02-07T18:57:33.111Z 8 160-179 51 51 51 87 87 87 F F F
69742024-02-07T18:57:33.111Z 9 180-199 57 57 57 94 94 94 F F F
69752024-02-07T18:57:33.115Z 10 200-219 57 57 57 94 94 94 F F F
69762024-02-07T18:57:33.115Z 11 220-239 62 62 62 99 99 99 F F F
69772024-02-07T18:57:33.115Z 12 240-259 63 63 63 100 100 100 F F F
69782024-02-07T18:57:33.115Z 13 260-279 49 49 49 85 85 85 F F F
69792024-02-07T18:57:33.115Z 14 280-299 63 63 63 100 100 100 F F F
69802024-02-07T18:57:33.115Z 15 300-319 63 63 63 100 100 100 F F F
69812024-02-07T18:57:33.115Z 16 320-339 62 62 62 98 98 98 F F F
69822024-02-07T18:57:33.115Z 17 340-359 63 63 63 100 100 100 F F F
69832024-02-07T18:57:33.115Z 18 360-379 63 63 63 100 100 100 F F F
69842024-02-07T18:57:33.115Z 19 380-399 60 60 60 96 96 96 F F F
69852024-02-07T18:57:33.115Z 20 400-419 63 63 63 100 100 100 F F F
69862024-02-07T18:57:33.115Z 21 420-439 62 62 62 98 98 98 F F F
69872024-02-07T18:57:33.115Z 22 440-459 63 63 63 100 100 100 F F F
69882024-02-07T18:57:33.115Z 23 460-479 58 58 58 95 95 95 F F F
69892024-02-07T18:57:33.115Z 24 480-499 62 62 62 99 99 99 F F F
69902024-02-07T18:57:33.116Z 25 500-519 62 62 62 99 99 99 F F F
69912024-02-07T18:57:33.116Z 26 520-539 62 62 62 99 99 99 F F F
69922024-02-07T18:57:33.116Z 27 540-559 62 62 62 99 99 99 F F F
69932024-02-07T18:57:33.116Z 28 560-579 62 62 62 99 99 99 F F F
69942024-02-07T18:57:33.116Z 29 580-599 60 60 60 96 96 96 F F F
69952024-02-07T18:57:33.116ZMax gen: 63, Max flush: 100
69962024-02-07T18:57:33.116Z
69972024-02-07T18:57:33.116ZBegin loop 32
69982024-02-07T18:57:33.116ZDownstairs to restart: 1
69992024-02-07T18:57:33.131Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.37955759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391}
70002024-02-07T18:57:33.131Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.37985236Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391}
70012024-02-07T18:57:33.131Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
70022024-02-07T18:57:33.131Z,"time":"2024-02-07T18:57:32.379885146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391}
70032024-02-07T18:57:33.137Z{"msg":"Crucible 48074494-86e0-4d42-a905-854aac2463b9 has session id: 3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.379934599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70042024-02-07T18:57:33.137Z{"msg":"Upstairs opts: Upstairs UUID: 48074494-86e0-4d42-a905-854aac2463b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.379979503Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70052024-02-07T18:57:33.137Z{"msg":"Crucible stats registered with UUID: 48074494-86e0-4d42-a905-854aac2463b9","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380007199Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70062024-02-07T18:57:33.137Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380047864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391}
70072024-02-07T18:57:33.137Z{"msg":"48074494-86e0-4d42-a905-854aac2463b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.3801464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70082024-02-07T18:57:33.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380183415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70092024-02-07T18:57:33.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380200983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70102024-02-07T18:57:33.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380218141Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70112024-02-07T18:57:33.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380253086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70122024-02-07T18:57:33.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380270733Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70132024-02-07T18:57:33.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.38029346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70142024-02-07T18:57:33.138Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380307698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70152024-02-07T18:57:33.138Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380322796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70162024-02-07T18:57:33.138Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380336645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70172024-02-07T18:57:33.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380461388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70182024-02-07T18:57:33.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380520949Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70192024-02-07T18:57:33.138Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:32.380619486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70202024-02-07T18:57:33.138Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:32.3806643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70212024-02-07T18:57:33.138Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:32.380776965Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70222024-02-07T18:57:33.138Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380804491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70232024-02-07T18:57:33.138Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380819519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70242024-02-07T18:57:33.138Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.380934973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70252024-02-07T18:57:33.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.381362316Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70262024-02-07T18:57:33.139Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.381391412Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70272024-02-07T18:57:33.139Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.38140747Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70282024-02-07T18:57:33.139Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.381436266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70292024-02-07T18:57:33.139Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382111485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70302024-02-07T18:57:33.139Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382172646Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70312024-02-07T18:57:33.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382650792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70322024-02-07T18:57:33.139Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382688036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70332024-02-07T18:57:33.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382704034Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70342024-02-07T18:57:33.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382771315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70352024-02-07T18:57:33.139Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382799041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70362024-02-07T18:57:33.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:32.382814449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70372024-02-07T18:57:34.135Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.382602147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70382024-02-07T18:57:34.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38280098Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70392024-02-07T18:57:34.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.383577875Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70402024-02-07T18:57:34.140Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38361598Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70412024-02-07T18:57:34.140Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.384174575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70422024-02-07T18:57:34.140Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38480245Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70432024-02-07T18:57:34.140Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.384852603Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70442024-02-07T18:57:34.140Z{"msg":"[0]R flush_numbers[0..12]: [97, 94, 96, 95, 95, 95, 96, 96, 87, 94, 94, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.384888248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70452024-02-07T18:57:34.140Z{"msg":"[0]R generation[0..12]: [61, 57, 60, 58, 58, 58, 60, 60, 51, 57, 57, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.384926313Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70462024-02-07T18:57:34.140Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38494506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70472024-02-07T18:57:34.140Z{"msg":"[1]R flush_numbers[0..12]: [97, 94, 96, 95, 95, 95, 96, 96, 87, 94, 94, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.384970867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70482024-02-07T18:57:34.140Z{"msg":"[1]R generation[0..12]: [61, 57, 60, 58, 58, 58, 60, 60, 51, 57, 57, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.384988694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70492024-02-07T18:57:34.140Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
70502024-02-07T18:57:34.140Zcrucible","level":30,"time":"2024-02-07T18:57:33.385018501Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70512024-02-07T18:57:34.140Z{"msg":"[2]R flush_numbers[0..12]: [97, 94, 96, 95, 95, 95, 96, 96, 87, 94, 94, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385037748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70522024-02-07T18:57:34.140Z{"msg":"[2]R generation[0..12]: [61, 57, 60, 58, 58, 58, 60, 60, 51, 57, 57, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385056085Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70532024-02-07T18:57:34.140Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385074173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70542024-02-07T18:57:34.140Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38509893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70552024-02-07T18:57:34.140Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385116637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70562024-02-07T18:57:34.140Z{"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385141324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70572024-02-07T18:57:34.140Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385158522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70582024-02-07T18:57:34.141Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385183388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70592024-02-07T18:57:34.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385198926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70602024-02-07T18:57:34.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385223273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70612024-02-07T18:57:34.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38524006Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70622024-02-07T18:57:34.141Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385264467Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70632024-02-07T18:57:34.141Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385280155Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70642024-02-07T18:57:34.141Z{"msg":"48074494-86e0-4d42-a905-854aac2463b9 is now active with session: 3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385303932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70652024-02-07T18:57:34.141Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.38531921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391,"session_id":"3ecb1d6d-e9c8-4eab-b9d8-dc2843231aa6"}
70662024-02-07T18:57:34.141Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:33.385343046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391}
70672024-02-07T18:57:34.141ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
70682024-02-07T18:57:34.141ZLoading write count information from file "/tmp/test_repair_verify.data"
70692024-02-07T18:57:34.141ZRun Repair workload
70702024-02-07T18:57:34.141Z01/30 Write block 539 len 40960 data: 9 9 9 11 6 5 4 4 6 7
70712024-02-07T18:57:34.145Z02/30 Read block 588 len 24576
70722024-02-07T18:57:34.145Z03/30 Read block 392 len 32768
70732024-02-07T18:57:34.145Z04/30 Read block 470 len 8192
70742024-02-07T18:57:34.145Z05/30 Write block 3 len 8192 data: 4 4
70752024-02-07T18:57:34.145Z06/30 Read block 199 len 20480
70762024-02-07T18:57:34.145Z07/30 Flush
70772024-02-07T18:57:34.148Z08/30 Flush
70782024-02-07T18:57:34.148Z09/30 Write block 241 len 8192 data: 10 10
70792024-02-07T18:57:34.148Z10/30 Flush
70802024-02-07T18:57:34.151Z11/30 Write block 569 len 16384 data: 5 6 6 4
70812024-02-07T18:57:34.151Z12/30 Read block 412 len 24576
70822024-02-07T18:57:34.152Z13/30 Read block 476 len 40960
70832024-02-07T18:57:34.154Z14/30 Write block 64 len 40960 data: 6 6 9 8 8 9 8 9 7 6
70842024-02-07T18:57:34.154Z15/30 Flush
70852024-02-07T18:57:34.157Z16/30 Write block 122 len 36864 data: 9 7 5 5 5 6 7 7 6
70862024-02-07T18:57:34.157Z17/30 Write block 417 len 12288 data: 9 8 5
70872024-02-07T18:57:34.157Z18/30 Read block 65 len 36864
70882024-02-07T18:57:34.161Z19/30 Write block 483 len 4096 data: 3
70892024-02-07T18:57:34.161Z20/30 Write block 240 len 28672 data: 10 11 11 11 10 7 7
70902024-02-07T18:57:34.161Z21/30 Write block 345 len 12288 data: 8 9 8
70912024-02-07T18:57:34.161Z22/30 Write block 229 len 4096 data: 5
70922024-02-07T18:57:34.161Z23/30 Read block 44 len 16384
70932024-02-07T18:57:34.164Z24/30 Read block 274 len 4096
70942024-02-07T18:57:34.164Z25/30 Read block 505 len 12288
70952024-02-07T18:57:34.164Z26/30 Read block 175 len 16384
70962024-02-07T18:57:34.164Z27/30 Write block 348 len 8192 data: 7 8
70972024-02-07T18:57:34.167Z28/30 Write block 263 len 28672 data: 7 7 6 4 3 3 3
70982024-02-07T18:57:34.168Z29/30 Read block 66 len 20480
70992024-02-07T18:57:34.168Z30/30 Read block 144 len 32768
71002024-02-07T18:57:34.168Z----------------------------------------------------------------
71012024-02-07T18:57:34.168Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
71022024-02-07T18:57:34.168ZStates: Active Active Active
71032024-02-07T18:57:34.168ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71042024-02-07T18:57:34.168Z 1 Acked 1000 Write 10 Done Done Done false
71052024-02-07T18:57:34.168Z 2 Acked 1001 Read 6 Done Done Done false
71062024-02-07T18:57:34.168Z 3 Acked 1002 Read 8 Done Done Done false
71072024-02-07T18:57:34.168Z 4 Acked 1003 Read 2 Done Done Done false
71082024-02-07T18:57:34.168Z 5 Acked 1004 Write 2 Done Done Done false
71092024-02-07T18:57:34.168Z 6 Acked 1005 Read 5 Done Done Done false
71102024-02-07T18:57:34.168Z 7 Acked 1006 Flush 0 Done Sent Done false
71112024-02-07T18:57:34.168Z 8 Acked 1007 Flush 0 Done Sent Done false
71122024-02-07T18:57:34.168Z 9 Acked 1008 Write 2 Done Sent Done false
71132024-02-07T18:57:34.168Z 10 Acked 1009 Flush 0 Done Sent Done false
71142024-02-07T18:57:34.168Z 11 Acked 1010 Write 4 Done Sent Done false
71152024-02-07T18:57:34.168Z 12 Acked 1011 Read 6 Done Sent Done false
71162024-02-07T18:57:34.168Z 13 Acked 1012 Read 10 Done Sent Done false
71172024-02-07T18:57:34.168Z 14 Acked 1013 Write 10 Done Sent Done false
71182024-02-07T18:57:34.168Z 15 Acked 1014 Flush 0 Done Sent Done false
71192024-02-07T18:57:34.168Z 16 Acked 1015 Write 9 Done Sent Done false
71202024-02-07T18:57:34.168Z 17 Acked 1016 Write 3 Done Sent Done false
71212024-02-07T18:57:34.169Z 18 Acked 1017 Read 9 Done Sent Done false
71222024-02-07T18:57:34.169Z 19 Acked 1018 Write 1 Done Sent Done false
71232024-02-07T18:57:34.169Z 20 Acked 1019 Write 7 Done Sent Done false
71242024-02-07T18:57:34.169Z 21 Acked 1020 Write 3 Done Sent Done false
71252024-02-07T18:57:34.169Z 22 Acked 1021 Write 1 Done Sent Done false
71262024-02-07T18:57:34.169Z 23 Acked 1022 Read 4 Done Sent Done false
71272024-02-07T18:57:34.169Z 24 Acked 1023 Read 1 Done Sent Done false
71282024-02-07T18:57:34.169Z 25 Acked 1024 Read 3 Done Sent Done false
71292024-02-07T18:57:34.169Z 26 Acked 1025 Read 4 Done Sent Done false
71302024-02-07T18:57:34.169Z 27 Acked 1026 Write 2 Done Sent Done false
71312024-02-07T18:57:34.169Z 28 Acked 1027 Write 7 Done Sent Done false
71322024-02-07T18:57:34.169Z 29 Acked 1028 Read 5 Done Sent Done false
71332024-02-07T18:57:34.169Z 30 Acked 1029 Read 8 Done Sent Done false
71342024-02-07T18:57:34.169Z STATES DS:0 DS:1 DS:2 TOTAL
71352024-02-07T18:57:34.169Z New 0 0 0 0
71362024-02-07T18:57:34.169Z Sent 0 24 0 24
71372024-02-07T18:57:34.169Z Done 30 6 30 66
71382024-02-07T18:57:34.169Z Skipped 0 0 0 0
71392024-02-07T18:57:34.169Z Error 0 0 0 0
71402024-02-07T18:57:34.169ZLast Flush: 1014 0 1014
71412024-02-07T18:57:34.169ZDownstairs last five completed:
71422024-02-07T18:57:34.169ZUpstairs last five completed: 30 29 28 27 26
71432024-02-07T18:57:34.173Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:33.416009089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1391}
71442024-02-07T18:57:34.173ZWrote out file "/tmp/test_repair_verify.data"
71452024-02-07T18:57:34.173Z
71462024-02-07T18:57:34.176ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
71472024-02-07T18:57:34.196ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
71482024-02-07T18:57:34.196Z 0 000-019 64 61 64 101 97 101 F T F <---
71492024-02-07T18:57:34.196Z 1 020-039 57 57 57 94 94 94 F F F
71502024-02-07T18:57:34.196Z 2 040-059 60 60 60 96 96 96 F F F
71512024-02-07T18:57:34.200Z 3 060-079 64 58 64 104 95 104 F F F <---
71522024-02-07T18:57:34.200Z 4 080-099 58 58 58 95 95 95 F F F
71532024-02-07T18:57:34.200Z 5 100-119 58 58 58 95 95 95 F F F
71542024-02-07T18:57:34.200Z 6 120-139 60 60 60 96 96 96 T F T <---
71552024-02-07T18:57:34.200Z 7 140-159 60 60 60 96 96 96 F F F
71562024-02-07T18:57:34.200Z 8 160-179 51 51 51 87 87 87 F F F
71572024-02-07T18:57:34.200Z 9 180-199 57 57 57 94 94 94 F F F
71582024-02-07T18:57:34.200Z 10 200-219 57 57 57 94 94 94 F F F
71592024-02-07T18:57:34.200Z 11 220-239 62 62 62 99 99 99 T F T <---
71602024-02-07T18:57:34.200Z 12 240-259 64 63 64 103 100 103 T F T <---
71612024-02-07T18:57:34.200Z 13 260-279 49 49 49 85 85 85 T F T <---
71622024-02-07T18:57:34.200Z 14 280-299 63 63 63 100 100 100 F F F
71632024-02-07T18:57:34.200Z 15 300-319 63 63 63 100 100 100 F F F
71642024-02-07T18:57:34.200Z 16 320-339 62 62 62 98 98 98 F F F
71652024-02-07T18:57:34.200Z 17 340-359 63 63 63 100 100 100 T F T <---
71662024-02-07T18:57:34.200Z 18 360-379 63 63 63 100 100 100 F F F
71672024-02-07T18:57:34.200Z 19 380-399 60 60 60 96 96 96 F F F
71682024-02-07T18:57:34.200Z 20 400-419 63 63 63 100 100 100 T F T <---
71692024-02-07T18:57:34.200Z 21 420-439 62 62 62 98 98 98 F F F
71702024-02-07T18:57:34.200Z 22 440-459 63 63 63 100 100 100 F F F
71712024-02-07T18:57:34.200Z 23 460-479 58 58 58 95 95 95 F F F
71722024-02-07T18:57:34.200Z 24 480-499 62 62 62 99 99 99 T F T <---
71732024-02-07T18:57:34.201Z 25 500-519 62 62 62 99 99 99 F F F
71742024-02-07T18:57:34.201Z 26 520-539 64 62 64 101 99 101 F T F <---
71752024-02-07T18:57:34.201Z 27 540-559 64 62 64 101 99 101 F T F <---
71762024-02-07T18:57:34.201Z 28 560-579 64 62 64 104 99 104 F F F <---
71772024-02-07T18:57:34.201Z 29 580-599 60 60 60 96 96 96 F F F
71782024-02-07T18:57:34.201ZMax gen: 64, Max flush: 104
71792024-02-07T18:57:34.201ZError: Difference in extent metadata found!
71802024-02-07T18:57:34.201ZA Difference in extent metadata is expected here
71812024-02-07T18:57:34.201ZOn loop 32
71822024-02-07T18:57:34.201Z
71832024-02-07T18:57:34.201ZVerifying data now
71842024-02-07T18:57:39.804ZNo change after verify
71852024-02-07T18:57:39.805ZLoop: 32 Downstairs dump after verify (and repair):
71862024-02-07T18:57:39.820ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
71872024-02-07T18:57:39.820Z 0 000-019 64 64 64 101 101 101 F F F
71882024-02-07T18:57:39.820Z 1 020-039 57 57 57 94 94 94 F F F
71892024-02-07T18:57:39.821Z 2 040-059 60 60 60 96 96 96 F F F
71902024-02-07T18:57:39.821Z 3 060-079 64 64 64 104 104 104 F F F
71912024-02-07T18:57:39.821Z 4 080-099 58 58 58 95 95 95 F F F
71922024-02-07T18:57:39.821Z 5 100-119 58 58 58 95 95 95 F F F
71932024-02-07T18:57:39.821Z 6 120-139 65 65 65 105 105 105 F F F
71942024-02-07T18:57:39.821Z 7 140-159 60 60 60 96 96 96 F F F
71952024-02-07T18:57:39.821Z 8 160-179 51 51 51 87 87 87 F F F
71962024-02-07T18:57:39.821Z 9 180-199 57 57 57 94 94 94 F F F
71972024-02-07T18:57:39.821Z 10 200-219 57 57 57 94 94 94 F F F
71982024-02-07T18:57:39.821Z 11 220-239 65 65 65 105 105 105 F F F
71992024-02-07T18:57:39.821Z 12 240-259 65 65 65 105 105 105 F F F
72002024-02-07T18:57:39.821Z 13 260-279 65 65 65 105 105 105 F F F
72012024-02-07T18:57:39.825Z 14 280-299 63 63 63 100 100 100 F F F
72022024-02-07T18:57:39.825Z 15 300-319 63 63 63 100 100 100 F F F
72032024-02-07T18:57:39.825Z 16 320-339 62 62 62 98 98 98 F F F
72042024-02-07T18:57:39.825Z 17 340-359 65 65 65 105 105 105 F F F
72052024-02-07T18:57:39.825Z 18 360-379 63 63 63 100 100 100 F F F
72062024-02-07T18:57:39.825Z 19 380-399 60 60 60 96 96 96 F F F
72072024-02-07T18:57:39.825Z 20 400-419 65 65 65 105 105 105 F F F
72082024-02-07T18:57:39.825Z 21 420-439 62 62 62 98 98 98 F F F
72092024-02-07T18:57:39.825Z 22 440-459 63 63 63 100 100 100 F F F
72102024-02-07T18:57:39.825Z 23 460-479 58 58 58 95 95 95 F F F
72112024-02-07T18:57:39.825Z 24 480-499 65 65 65 105 105 105 F F F
72122024-02-07T18:57:39.825Z 25 500-519 62 62 62 99 99 99 F F F
72132024-02-07T18:57:39.825Z 26 520-539 64 64 64 101 101 101 F F F
72142024-02-07T18:57:39.825Z 27 540-559 64 64 64 101 101 101 F F F
72152024-02-07T18:57:39.825Z 28 560-579 64 64 64 104 104 104 F F F
72162024-02-07T18:57:39.825Z 29 580-599 60 60 60 96 96 96 F F F
72172024-02-07T18:57:39.825ZMax gen: 65, Max flush: 105
72182024-02-07T18:57:39.825Z
72192024-02-07T18:57:39.825ZBegin loop 33
72202024-02-07T18:57:39.825ZDownstairs to restart: 1
72212024-02-07T18:57:39.845Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09138442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399}
72222024-02-07T18:57:39.845Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.091643775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399}
72232024-02-07T18:57:39.845Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09168391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399}
72242024-02-07T18:57:39.850Z{"msg":"Crucible 1c937c3a-bc88-422d-853e-bce030675300 has session id: 623f8ec3-4db7-4aea-baf3-50b144b995f0","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.091712166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72252024-02-07T18:57:39.850Z{"msg":"Upstairs opts: Upstairs UUID: 1c937c3a-bc88-422d-853e-bce030675300, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.091748271Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72262024-02-07T18:57:39.850Z{"msg":"Crucible stats registered with UUID: 1c937c3a-bc88-422d-853e-bce030675300","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.091791745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72272024-02-07T18:57:39.850ZCrucible runtime is spawned
72282024-02-07T18:57:39.850Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.091929186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399}
72292024-02-07T18:57:39.850Z{"msg":"1c937c3a-bc88-422d-853e-bce030675300 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092017254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72302024-02-07T18:57:39.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092056049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72312024-02-07T18:57:39.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092083896Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72322024-02-07T18:57:39.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092115771Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72332024-02-07T18:57:39.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092139058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72342024-02-07T18:57:39.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092169354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72352024-02-07T18:57:39.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.0921961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72362024-02-07T18:57:39.851Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092223107Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72372024-02-07T18:57:39.851Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092282829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72382024-02-07T18:57:39.851Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092412941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72392024-02-07T18:57:39.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092466804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72402024-02-07T18:57:39.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092506648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72412024-02-07T18:57:39.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.092547383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72422024-02-07T18:57:39.851Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.093092309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72432024-02-07T18:57:39.851Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.093122425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72442024-02-07T18:57:39.851Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.093138253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72452024-02-07T18:57:39.851Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.093168589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72462024-02-07T18:57:39.851Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.093669721Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72472024-02-07T18:57:39.851Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09382693Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72482024-02-07T18:57:39.851Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094318253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72492024-02-07T18:57:39.851Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094359438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72502024-02-07T18:57:39.851Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094389574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72512024-02-07T18:57:39.851Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094426799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72522024-02-07T18:57:39.851Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094514057Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72532024-02-07T18:57:39.851Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094550182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72542024-02-07T18:57:39.851Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09456615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72552024-02-07T18:57:39.851Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.094580408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72562024-02-07T18:57:39.851Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095044995Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72572024-02-07T18:57:39.851Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09552819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72582024-02-07T18:57:39.851Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095567414Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72592024-02-07T18:57:39.852Z{"msg":"[0]R flush_numbers[0..12]: [101, 94, 96, 104, 95, 95, 105, 96, 87, 94, 94, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095584902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72602024-02-07T18:57:39.852Z{"msg":"[0]R generation[0..12]: [64, 57, 60, 64, 58, 58, 65, 60, 51, 57, 57, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095607029Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72612024-02-07T18:57:39.852Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095622137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72622024-02-07T18:57:39.852Z{"msg":"[1]R flush_numbers[0..12]: [101, 94, 96, 104, 95, 95, 105, 96, 87, 94, 94, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095643434Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72632024-02-07T18:57:39.852Z{"msg":"[1]R generation[0..12]: [64, 57, 60, 64, 58, 58, 65, 60, 51, 57, 57, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095657732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72642024-02-07T18:57:39.852Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095683849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
72652024-02-07T18:57:39.852Z623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72662024-02-07T18:57:39.852Z{"msg":"[2]R flush_numbers[0..12]: [101, 94, 96, 104, 95, 95, 105, 96, 87, 94, 94, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095713365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72672024-02-07T18:57:39.852Z{"msg":"[2]R generation[0..12]: [64, 57, 60, 64, 58, 58, 65, 60, 51, 57, 57, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095727433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72682024-02-07T18:57:39.852Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09574883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72692024-02-07T18:57:39.852Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095763068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72702024-02-07T18:57:39.852Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095777006Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72712024-02-07T18:57:39.852Z{"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095797043Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72722024-02-07T18:57:39.852Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095810941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72732024-02-07T18:57:39.852Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095830349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72742024-02-07T18:57:39.852Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095843147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72752024-02-07T18:57:39.852Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095864664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72762024-02-07T18:57:39.852Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095878422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72772024-02-07T18:57:39.852Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095901519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72782024-02-07T18:57:39.852Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095917317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72792024-02-07T18:57:39.852Z{"msg":"1c937c3a-bc88-422d-853e-bce030675300 is now active with session: 623f8ec3-4db7-4aea-baf3-50b144b995f0","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095930555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72802024-02-07T18:57:39.852Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.095953152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399,"session_id":"623f8ec3-4db7-4aea-baf3-50b144b995f0"}
72812024-02-07T18:57:39.852Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:39.09597142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399}
72822024-02-07T18:57:39.856ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
72832024-02-07T18:57:39.856ZLoading write count information from file "/tmp/test_repair_verify.data"
72842024-02-07T18:57:39.856ZRun Repair workload
72852024-02-07T18:57:39.856Z01/30 Write block 527 len 16384 data: 7 7 7 6
72862024-02-07T18:57:39.856Z02/30 Read block 480 len 36864
72872024-02-07T18:57:39.856Z03/30 Write block 93 len 40960 data: 6 8 7 7 7 6 5 4 4 4
72882024-02-07T18:57:39.856Z04/30 Flush
72892024-02-07T18:57:39.856Z05/30 Write block 287 len 28672 data: 9 9 8 9 7 5 6
72902024-02-07T18:57:39.856Z06/30 Write block 534 len 4096 data: 7
72912024-02-07T18:57:39.860Z07/30 Read block 554 len 36864
72922024-02-07T18:57:39.861Z08/30 Write block 239 len 16384 data: 8 11 12 12
72932024-02-07T18:57:39.861Z09/30 Read block 50 len 24576
72942024-02-07T18:57:39.861Z10/30 Write block 305 len 8192 data: 5 4
72952024-02-07T18:57:39.861Z11/30 Read block 223 len 12288
72962024-02-07T18:57:39.871Z12/30 Read block 294 len 20480
72972024-02-07T18:57:39.871Z13/30 Write block 505 len 32768 data: 6 5 5 5 5 7 6 9
72982024-02-07T18:57:39.871Z14/30 Read block 90 len 8192
72992024-02-07T18:57:39.871Z15/30 Write block 439 len 36864 data: 7 7 7 9 9 9 8 8 6
73002024-02-07T18:57:39.871Z16/30 Write block 295 len 40960 data: 6 5 7 6 4 4 3 4 3 5
73012024-02-07T18:57:39.871Z17/30 Read block 553 len 20480
73022024-02-07T18:57:39.887Z18/30 Write block 502 len 20480 data: 5 6 6 7 6
73032024-02-07T18:57:39.887Z19/30 Read block 56 len 8192
73042024-02-07T18:57:39.887Z20/30 Write block 7 len 32768 data: 7 6 7 6 6 6 6 7
73052024-02-07T18:57:39.887Z21/30 Write block 12 len 28672 data: 7 7 8 6 9 10 8
73062024-02-07T18:57:39.887Z22/30 Write block 325 len 40960 data: 5 4 3 3 3 4 5 5 6 6
73072024-02-07T18:57:39.887Z23/30 Read block 457 len 28672
73082024-02-07T18:57:39.887Z24/30 Write block 310 len 40960 data: 6 6 4 5 6 9 11 9 9 7
73092024-02-07T18:57:39.887Z25/30 Write block 540 len 8192 data: 10 10
73102024-02-07T18:57:39.887Z26/30 Read block 354 len 8192
73112024-02-07T18:57:39.887Z27/30 Read block 507 len 36864
73122024-02-07T18:57:39.888Z28/30 Read block 371 len 16384
73132024-02-07T18:57:39.888Z29/30 Read block 579 len 16384
73142024-02-07T18:57:39.888Z30/30 Write block 173 len 12288 data: 7 6 5
73152024-02-07T18:57:39.888Z----------------------------------------------------------------
73162024-02-07T18:57:39.888Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
73172024-02-07T18:57:39.888ZStates: Active Active Active
73182024-02-07T18:57:39.888ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
73192024-02-07T18:57:39.888Z 1 Acked 1000 Write 4 Done Done Done false
73202024-02-07T18:57:39.888Z 2 Acked 1001 Read 9 Done Done Done false
73212024-02-07T18:57:39.888Z 3 Acked 1002 Write 10 Done Sent Done false
73222024-02-07T18:57:39.888Z 4 Acked 1003 Flush 0 Done Sent Done false
73232024-02-07T18:57:39.888Z 5 Acked 1004 Write 7 Done Sent Done false
73242024-02-07T18:57:39.888Z 6 Acked 1005 Write 1 Done Sent Done false
73252024-02-07T18:57:39.888Z 7 Acked 1006 Read 9 Done Sent Done false
73262024-02-07T18:57:39.888Z 8 Acked 1007 Write 4 Done Sent Done false
73272024-02-07T18:57:39.888Z 9 Acked 1008 Read 6 Done Sent Done false
73282024-02-07T18:57:39.888Z 10 Acked 1009 Write 2 Done Sent Done false
73292024-02-07T18:57:39.888Z 11 Acked 1010 Read 3 Done Sent Done false
73302024-02-07T18:57:39.888Z 12 Acked 1011 Read 5 Done Sent Done false
73312024-02-07T18:57:39.888Z 13 Acked 1012 Write 8 Done Sent Done false
73322024-02-07T18:57:39.888Z 14 Acked 1013 Read 2 Done Sent Done false
73332024-02-07T18:57:39.888Z 15 Acked 1014 Write 9 Done Sent Done false
73342024-02-07T18:57:39.888Z 16 Acked 1015 Write 10 Done Sent Done false
73352024-02-07T18:57:39.888Z 17 Acked 1016 Read 5 Done Sent Done false
73362024-02-07T18:57:39.888Z 18 Acked 1017 Write 5 Done Sent Done false
73372024-02-07T18:57:39.889Z 19 Acked 1018 Read 2 Done Sent Done false
73382024-02-07T18:57:39.889Z 20 Acked 1019 Write 8 Done Sent Done false
73392024-02-07T18:57:39.889Z 21 Acked 1020 Write 7 Done Sent Done false
73402024-02-07T18:57:39.889Z 22 Acked 1021 Write 10 Done Sent Done false
73412024-02-07T18:57:39.889Z 23 Acked 1022 Read 7 Done Sent Done false
73422024-02-07T18:57:39.889Z 24 Acked 1023 Write 10 Done Sent Done false
73432024-02-07T18:57:39.889Z 25 Acked 1024 Write 2 Done Sent Done false
73442024-02-07T18:57:39.889Z 26 Acked 1025 Read 2 Done Sent Done false
73452024-02-07T18:57:39.889Z 27 Acked 1026 Read 9 Done Sent Done false
73462024-02-07T18:57:39.889Z 28 Acked 1027 Read 4 Done Sent Done false
73472024-02-07T18:57:39.889Z 29 Acked 1028 Read 4 Done Sent Done false
73482024-02-07T18:57:39.889Z 30 Acked 1029 Write 3 Sent Sent Sent false
73492024-02-07T18:57:39.889Z STATES DS:0 DS:1 DS:2 TOTAL
73502024-02-07T18:57:39.889Z New 0 0 0 0
73512024-02-07T18:57:39.889Z Sent 1 28 1 30
73522024-02-07T18:57:39.889Z Done 29 2 29 60
73532024-02-07T18:57:39.889Z Skipped 0 0 0 0
73542024-02-07T18:57:39.889Z Error 0 0 0 0
73552024-02-07T18:57:39.889ZLast Flush: 1003 0 1003
73562024-02-07T18:57:39.889ZDownstairs last five completed:
73572024-02-07T18:57:39.889ZUpstairs last five completed: 30 29 28 27 26
73582024-02-07T18:57:39.889Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:39.124917605Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1399}
73592024-02-07T18:57:39.889ZWrote out file "/tmp/test_repair_verify.data"
73602024-02-07T18:57:39.893Z
73612024-02-07T18:57:39.893ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
73622024-02-07T18:57:39.905ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73632024-02-07T18:57:39.905Z 0 000-019 64 64 64 101 101 101 T F T <---
73642024-02-07T18:57:39.905Z 1 020-039 57 57 57 94 94 94 F F F
73652024-02-07T18:57:39.906Z 2 040-059 60 60 60 96 96 96 F F F
73662024-02-07T18:57:39.906Z 3 060-079 64 64 64 104 104 104 F F F
73672024-02-07T18:57:39.906Z 4 080-099 66 58 66 106 95 106 F F F <---
73682024-02-07T18:57:39.909Z 5 100-119 66 58 66 106 95 106 F F F <---
73692024-02-07T18:57:39.909Z 6 120-139 65 65 65 105 105 105 F F F
73702024-02-07T18:57:39.909Z 7 140-159 60 60 60 96 96 96 F F F
73712024-02-07T18:57:39.909Z 8 160-179 51 51 51 87 87 87 T F T <---
73722024-02-07T18:57:39.909Z 9 180-199 57 57 57 94 94 94 F F F
73732024-02-07T18:57:39.909Z 10 200-219 57 57 57 94 94 94 F F F
73742024-02-07T18:57:39.909Z 11 220-239 65 65 65 105 105 105 T F T <---
73752024-02-07T18:57:39.909Z 12 240-259 65 65 65 105 105 105 T F T <---
73762024-02-07T18:57:39.909Z 13 260-279 65 65 65 105 105 105 F F F
73772024-02-07T18:57:39.909Z 14 280-299 63 63 63 100 100 100 T F T <---
73782024-02-07T18:57:39.909Z 15 300-319 63 63 63 100 100 100 T F T <---
73792024-02-07T18:57:39.909Z 16 320-339 62 62 62 98 98 98 T F T <---
73802024-02-07T18:57:39.909Z 17 340-359 65 65 65 105 105 105 F F F
73812024-02-07T18:57:39.909Z 18 360-379 63 63 63 100 100 100 F F F
73822024-02-07T18:57:39.909Z 19 380-399 60 60 60 96 96 96 F F F
73832024-02-07T18:57:39.909Z 20 400-419 65 65 65 105 105 105 F F F
73842024-02-07T18:57:39.909Z 21 420-439 62 62 62 98 98 98 T F T <---
73852024-02-07T18:57:39.910Z 22 440-459 63 63 63 100 100 100 T F T <---
73862024-02-07T18:57:39.910Z 23 460-479 58 58 58 95 95 95 F F F
73872024-02-07T18:57:39.910Z 24 480-499 65 65 65 105 105 105 F F F
73882024-02-07T18:57:39.910Z 25 500-519 62 62 62 99 99 99 T F T <---
73892024-02-07T18:57:39.910Z 26 520-539 66 64 66 106 101 106 T T T <---
73902024-02-07T18:57:39.910Z 27 540-559 64 64 64 101 101 101 T F T <---
73912024-02-07T18:57:39.910Z 28 560-579 64 64 64 104 104 104 F F F
73922024-02-07T18:57:39.910Z 29 580-599 60 60 60 96 96 96 F F F
73932024-02-07T18:57:39.910ZMax gen: 66, Max flush: 106
73942024-02-07T18:57:39.910ZError: Difference in extent metadata found!
73952024-02-07T18:57:39.910ZA Difference in extent metadata is expected here
73962024-02-07T18:57:39.910ZOn loop 33
73972024-02-07T18:57:39.910Z
73982024-02-07T18:57:39.913ZVerifying data now
73992024-02-07T18:57:45.645ZNo change after verify
74002024-02-07T18:57:45.645ZLoop: 33 Downstairs dump after verify (and repair):
74012024-02-07T18:57:45.656ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74022024-02-07T18:57:45.657Z 0 000-019 67 67 67 107 107 107 F F F
74032024-02-07T18:57:45.657Z 1 020-039 57 57 57 94 94 94 F F F
74042024-02-07T18:57:45.657Z 2 040-059 60 60 60 96 96 96 F F F
74052024-02-07T18:57:45.657Z 3 060-079 64 64 64 104 104 104 F F F
74062024-02-07T18:57:45.657Z 4 080-099 66 66 66 106 106 106 F F F
74072024-02-07T18:57:45.657Z 5 100-119 66 66 66 106 106 106 F F F
74082024-02-07T18:57:45.657Z 6 120-139 65 65 65 105 105 105 F F F
74092024-02-07T18:57:45.657Z 7 140-159 60 60 60 96 96 96 F F F
74102024-02-07T18:57:45.657Z 8 160-179 67 67 67 107 107 107 F F F
74112024-02-07T18:57:45.657Z 9 180-199 57 57 57 94 94 94 F F F
74122024-02-07T18:57:45.657Z 10 200-219 57 57 57 94 94 94 F F F
74132024-02-07T18:57:45.657Z 11 220-239 67 67 67 107 107 107 F F F
74142024-02-07T18:57:45.657Z 12 240-259 67 67 67 107 107 107 F F F
74152024-02-07T18:57:45.657Z 13 260-279 65 65 65 105 105 105 F F F
74162024-02-07T18:57:45.657Z 14 280-299 67 67 67 107 107 107 F F F
74172024-02-07T18:57:45.661Z 15 300-319 67 67 67 107 107 107 F F F
74182024-02-07T18:57:45.661Z 16 320-339 67 67 67 107 107 107 F F F
74192024-02-07T18:57:45.661Z 17 340-359 65 65 65 105 105 105 F F F
74202024-02-07T18:57:45.661Z 18 360-379 63 63 63 100 100 100 F F F
74212024-02-07T18:57:45.661Z 19 380-399 60 60 60 96 96 96 F F F
74222024-02-07T18:57:45.661Z 20 400-419 65 65 65 105 105 105 F F F
74232024-02-07T18:57:45.661Z 21 420-439 67 67 67 107 107 107 F F F
74242024-02-07T18:57:45.661Z 22 440-459 67 67 67 107 107 107 F F F
74252024-02-07T18:57:45.661Z 23 460-479 58 58 58 95 95 95 F F F
74262024-02-07T18:57:45.661Z 24 480-499 65 65 65 105 105 105 F F F
74272024-02-07T18:57:45.661Z 25 500-519 67 67 67 107 107 107 F F F
74282024-02-07T18:57:45.661Z 26 520-539 67 67 67 107 107 107 F F F
74292024-02-07T18:57:45.661Z 27 540-559 67 67 67 107 107 107 F F F
74302024-02-07T18:57:45.661Z 28 560-579 64 64 64 104 104 104 F F F
74312024-02-07T18:57:45.661Z 29 580-599 60 60 60 96 96 96 F F F
74322024-02-07T18:57:45.661ZMax gen: 67, Max flush: 107
74332024-02-07T18:57:45.661Z
74342024-02-07T18:57:45.661ZBegin loop 34
74352024-02-07T18:57:45.661ZDownstairs to restart: 0
74362024-02-07T18:57:45.677Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.923823659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407}
74372024-02-07T18:57:45.683Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924148005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407}
74382024-02-07T18:57:45.683ZCrucible runtime is spawned
74392024-02-07T18:57:45.683Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924214026Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407}
74402024-02-07T18:57:45.683Z{"msg":"Crucible 442e2a67-5705-4667-ba70-583dedf884c1 has session id: 28c66b39-3987-4d26-8b51-bcd4a86f2a61","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924252531Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74412024-02-07T18:57:45.683Z{"msg":"Upstairs opts: Upstairs UUID: 442e2a67-5705-4667-ba70-583dedf884c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924297495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74422024-02-07T18:57:45.683Z{"msg":"Crucible stats registered with UUID: 442e2a67-5705-4667-ba70-583dedf884c1","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.92433475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74432024-02-07T18:57:45.683Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924353047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407}
74442024-02-07T18:57:45.683Z{"msg":"442e2a67-5705-4667-ba70-583dedf884c1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924371755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74452024-02-07T18:57:45.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924400571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74462024-02-07T18:57:45.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924420248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74472024-02-07T18:57:45.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924439695Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74482024-02-07T18:57:45.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924470491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74492024-02-07T18:57:45.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924486099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74502024-02-07T18:57:45.684Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924507956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74512024-02-07T18:57:45.684Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924541602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74522024-02-07T18:57:45.684Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924581566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74532024-02-07T18:57:45.684Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.92462752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74542024-02-07T18:57:45.684Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:44.924669244Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74552024-02-07T18:57:45.684Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:44.924711529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74562024-02-07T18:57:45.684Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:44.924754773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74572024-02-07T18:57:45.684Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.92477622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74582024-02-07T18:57:45.684Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924805296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74592024-02-07T18:57:45.684Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924823643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74602024-02-07T18:57:45.684Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924857979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74612024-02-07T18:57:45.684Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.924903953Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74622024-02-07T18:57:45.684Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.925415454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74632024-02-07T18:57:45.685Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.925461897Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74642024-02-07T18:57:45.685Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.925492233Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74652024-02-07T18:57:45.685Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.925522249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74662024-02-07T18:57:45.685Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926045038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74672024-02-07T18:57:45.685Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926087093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74682024-02-07T18:57:45.685Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926631609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74692024-02-07T18:57:45.685Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926669834Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74702024-02-07T18:57:45.685Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926685482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74712024-02-07T18:57:45.685Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.92670152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74722024-02-07T18:57:45.685Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926733745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74732024-02-07T18:57:45.685Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:44.926754522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74742024-02-07T18:57:46.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.925450308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74752024-02-07T18:57:46.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.925557533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74762024-02-07T18:57:46.684Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.926441424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74772024-02-07T18:57:46.684Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.926480379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74782024-02-07T18:57:46.684Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927022835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74792024-02-07T18:57:46.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927620925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74802024-02-07T18:57:46.684Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927661649Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74812024-02-07T18:57:46.684Z{"msg":"[0]R flush_numbers[0..12]: [107, 94, 96, 104, 106, 106, 105, 96, 107, 94, 94, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927708623Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74822024-02-07T18:57:46.684Z{"msg":"[0]R generation[0..12]: [67, 57, 60, 64, 66, 66, 65, 60, 67, 57, 57, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927732769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74832024-02-07T18:57:46.685Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927748197Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74842024-02-07T18:57:46.685Z{"msg":"[1]R flush_numbers[0..12]: [107, 94, 96, 104, 106, 106, 105, 96, 107, 94, 94, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927769714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74852024-02-07T18:57:46.685Z{"msg":"[1]R generation[0..12]: [67, 57, 60, 64, 66, 66, 65, 60, 67, 57, 57, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927784912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74862024-02-07T18:57:46.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.92779929Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74872024-02-07T18:57:46.685ZWait for a query_work_queue command to finish before sending IO
74882024-02-07T18:57:46.685Z{"msg":"[2]R flush_numbers[0..12]: [107, 94, 96, 104, 106, 106, 105, 96, 107, 94, 94, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927824877Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74892024-02-07T18:57:46.685Z{"msg":"[2]R generation[0..12]: [67, 57, 60, 64, 66, 66, 65, 60, 67, 57, 57, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927839675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74902024-02-07T18:57:46.685Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927862762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74912024-02-07T18:57:46.685Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.92787845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74922024-02-07T18:57:46.685Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927912815Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74932024-02-07T18:57:46.685Z{"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927931363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74942024-02-07T18:57:46.685Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927958059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74952024-02-07T18:57:46.685Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.927993234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74962024-02-07T18:57:46.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.928026939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74972024-02-07T18:57:46.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.928062135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74982024-02-07T18:57:46.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.92809829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
74992024-02-07T18:57:46.685Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75002024-02-07T18:57:46.685Z"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.928138015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
75012024-02-07T18:57:46.685Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.92816951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
75022024-02-07T18:57:46.685Z{"msg":"442e2a67-5705-4667-ba70-583dedf884c1 is now active with session: 28c66b39-3987-4d26-8b51-bcd4a86f2a61","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.928186348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
75032024-02-07T18:57:46.685Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.928205915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407,"session_id":"28c66b39-3987-4d26-8b51-bcd4a86f2a61"}
75042024-02-07T18:57:46.685Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:45.928220024Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407}
75052024-02-07T18:57:46.685ZLoading write count information from file "/tmp/test_repair_verify.data"
75062024-02-07T18:57:46.686ZRun Repair workload
75072024-02-07T18:57:46.686Z01/30 Write block 235 len 40960 data: 13 11 10 9 9 12 13 13 12 11
75082024-02-07T18:57:46.689Z02/30 Write block 457 len 40960 data: 5 6 5 4 4 4 5 5 5 5
75092024-02-07T18:57:46.689Z03/30 Write block 143 len 20480 data: 7 6 6 8 8
75102024-02-07T18:57:46.689Z04/30 Write block 440 len 8192 data: 8 8
75112024-02-07T18:57:46.689Z05/30 Read block 263 len 8192
75122024-02-07T18:57:46.689Z06/30 Read block 79 len 24576
75132024-02-07T18:57:46.689Z07/30 Read block 337 len 12288
75142024-02-07T18:57:46.690Z08/30 Write block 487 len 8192 data: 2 4
75152024-02-07T18:57:46.696Z09/30 Flush
75162024-02-07T18:57:46.696Z10/30 Write block 428 len 20480 data: 9 8 6 6 4
75172024-02-07T18:57:46.699Z11/30 Write block 555 len 28672 data: 5 5 4 5 6 5 3
75182024-02-07T18:57:46.699Z12/30 Flush
75192024-02-07T18:57:46.699Z13/30 Write block 141 len 32768 data: 7 7 8 7 7 9 9 9
75202024-02-07T18:57:46.699Z14/30 Write block 230 len 40960 data: 5 6 9 9 10 14 12 11 10 10
75212024-02-07T18:57:46.699Z15/30 Write block 215 len 20480 data: 4 3 3 4 4
75222024-02-07T18:57:46.699Z16/30 Read block 480 len 36864
75232024-02-07T18:57:46.702Z17/30 Read block 152 len 24576
75242024-02-07T18:57:46.702Z18/30 Read block 273 len 4096
75252024-02-07T18:57:46.702Z19/30 Read block 217 len 20480
75262024-02-07T18:57:46.703Z20/30 Read block 41 len 24576
75272024-02-07T18:57:46.707Z21/30 Read block 554 len 32768
75282024-02-07T18:57:46.707Z22/30 Read block 350 len 8192
75292024-02-07T18:57:46.707Z23/30 Read block 572 len 16384
75302024-02-07T18:57:46.707Z24/30 Write block 168 len 20480 data: 9 8 7 7 7
75312024-02-07T18:57:46.707Z25/30 Flush
75322024-02-07T18:57:46.710Z26/30 Write block 409 len 20480 data: 6 9 10 6 9
75332024-02-07T18:57:46.710Z27/30 Read block 77 len 40960
75342024-02-07T18:57:46.714Z28/30 Read block 368 len 4096
75352024-02-07T18:57:46.715Z29/30 Write block 122 len 16384 data: 10 8 6 6
75362024-02-07T18:57:46.715Z30/30 Read block 149 len 16384
75372024-02-07T18:57:46.715Z----------------------------------------------------------------
75382024-02-07T18:57:46.715Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
75392024-02-07T18:57:46.715ZStates: Active Active Active
75402024-02-07T18:57:46.715ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
75412024-02-07T18:57:46.715Z 1 Acked 1000 Write 10 Done Done Done false
75422024-02-07T18:57:46.715Z 2 Acked 1001 Write 10 Done Done Done false
75432024-02-07T18:57:46.715Z 3 Acked 1002 Write 5 Done Done Done false
75442024-02-07T18:57:46.715Z 4 Acked 1003 Write 2 Done Done Done false
75452024-02-07T18:57:46.715Z 5 Acked 1004 Read 2 Done Done Done false
75462024-02-07T18:57:46.715Z 6 Acked 1005 Read 6 Done Done Done false
75472024-02-07T18:57:46.715Z 7 Acked 1006 Read 3 Sent Done Done false
75482024-02-07T18:57:46.715Z 8 Acked 1007 Write 2 Sent Done Done false
75492024-02-07T18:57:46.715Z 9 Acked 1008 Flush 0 Sent Done Done false
75502024-02-07T18:57:46.715Z 10 Acked 1009 Write 5 Sent Done Done false
75512024-02-07T18:57:46.715Z 11 Acked 1010 Write 7 Sent Done Done false
75522024-02-07T18:57:46.715Z 12 Acked 1011 Flush 0 Sent Done Done false
75532024-02-07T18:57:46.715Z 13 Acked 1012 Write 8 Sent Done Done false
75542024-02-07T18:57:46.715Z 14 Acked 1013 Write 10 Sent Done Done false
75552024-02-07T18:57:46.715Z 15 Acked 1014 Write 5 Sent Done Done false
75562024-02-07T18:57:46.716Z 16 Acked 1015 Read 9 Sent Done Done false
75572024-02-07T18:57:46.716Z 17 Acked 1016 Read 6 Sent Done Done false
75582024-02-07T18:57:46.716Z 18 Acked 1017 Read 1 Sent Done Done false
75592024-02-07T18:57:46.716Z 19 Acked 1018 Read 5 Sent Done Done false
75602024-02-07T18:57:46.716Z 20 Acked 1019 Read 6 Sent Done Done false
75612024-02-07T18:57:46.716Z 21 Acked 1020 Read 8 Sent Done Done false
75622024-02-07T18:57:46.716Z 22 Acked 1021 Read 2 Sent Done Done false
75632024-02-07T18:57:46.716Z 23 Acked 1022 Read 4 Sent Done Done false
75642024-02-07T18:57:46.716Z 24 Acked 1023 Write 5 Sent Done Done false
75652024-02-07T18:57:46.716Z 25 Acked 1024 Flush 0 Sent Done Done false
75662024-02-07T18:57:46.716Z 26 Acked 1025 Write 5 Sent Done Done false
75672024-02-07T18:57:46.716Z 27 Acked 1026 Read 10 Sent Done Done false
75682024-02-07T18:57:46.716Z 28 Acked 1027 Read 1 Sent Done Done false
75692024-02-07T18:57:46.716Z 29 Acked 1028 Write 4 Sent Done Done false
75702024-02-07T18:57:46.716Z 30 Acked 1029 Read 4 Sent Done Sent false
75712024-02-07T18:57:46.716Z STATES DS:0 DS:1 DS:2 TOTAL
75722024-02-07T18:57:46.716Z New 0 0 0 0
75732024-02-07T18:57:46.716Z Sent 24 0 1 25
75742024-02-07T18:57:46.716Z Done 6 30 29 65
75752024-02-07T18:57:46.716Z Skipped 0 0 0 0
75762024-02-07T18:57:46.716Z Error 0 0 0 0
75772024-02-07T18:57:46.716ZLast Flush: 0 1024 1024
75782024-02-07T18:57:46.716ZDownstairs last five completed:
75792024-02-07T18:57:46.716ZUpstairs last five completed: 30 29 28 27 26
75802024-02-07T18:57:46.717Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:45.959833958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1407}
75812024-02-07T18:57:46.717ZWrote out file "/tmp/test_repair_verify.data"
75822024-02-07T18:57:46.720Z
75832024-02-07T18:57:46.723ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
75842024-02-07T18:57:46.738ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75852024-02-07T18:57:46.738Z 0 000-019 67 67 67 107 107 107 F F F
75862024-02-07T18:57:46.738Z 1 020-039 57 57 57 94 94 94 F F F
75872024-02-07T18:57:46.739Z 2 040-059 60 60 60 96 96 96 F F F
75882024-02-07T18:57:46.739Z 3 060-079 64 64 64 104 104 104 F F F
75892024-02-07T18:57:46.739Z 4 080-099 66 66 66 106 106 106 F F F
75902024-02-07T18:57:46.739Z 5 100-119 66 66 66 106 106 106 F F F
75912024-02-07T18:57:46.739Z 6 120-139 65 65 65 105 105 105 F T T <---
75922024-02-07T18:57:46.739Z 7 140-159 60 68 68 96 110 110 T F F <---
75932024-02-07T18:57:46.739Z 8 160-179 67 68 68 107 110 110 F F F <---
75942024-02-07T18:57:46.739Z 9 180-199 57 57 57 94 94 94 F F F
75952024-02-07T18:57:46.739Z 10 200-219 57 68 68 94 110 110 F F F <---
75962024-02-07T18:57:46.739Z 11 220-239 67 68 68 107 110 110 T F F <---
75972024-02-07T18:57:46.739Z 12 240-259 67 68 68 107 108 108 T F F <---
75982024-02-07T18:57:46.739Z 13 260-279 65 65 65 105 105 105 F F F
75992024-02-07T18:57:46.739Z 14 280-299 67 67 67 107 107 107 F F F
76002024-02-07T18:57:46.739Z 15 300-319 67 67 67 107 107 107 F F F
76012024-02-07T18:57:46.739Z 16 320-339 67 67 67 107 107 107 F F F
76022024-02-07T18:57:46.746Z 17 340-359 65 65 65 105 105 105 F F F
76032024-02-07T18:57:46.746Z 18 360-379 63 63 63 100 100 100 F F F
76042024-02-07T18:57:46.746Z 19 380-399 60 60 60 96 96 96 F F F
76052024-02-07T18:57:46.746Z 20 400-419 65 65 65 105 105 105 F T T <---
76062024-02-07T18:57:46.746Z 21 420-439 67 68 68 107 109 109 F F F <---
76072024-02-07T18:57:46.746Z 22 440-459 67 68 68 107 108 108 T F F <---
76082024-02-07T18:57:46.746Z 23 460-479 58 68 68 95 108 108 T F F <---
76092024-02-07T18:57:46.747Z 24 480-499 65 68 68 105 108 108 F F F <---
76102024-02-07T18:57:46.747Z 25 500-519 67 67 67 107 107 107 F F F
76112024-02-07T18:57:46.747Z 26 520-539 67 67 67 107 107 107 F F F
76122024-02-07T18:57:46.747Z 27 540-559 67 68 68 107 109 109 F F F <---
76132024-02-07T18:57:46.747Z 28 560-579 64 68 68 104 109 109 F F F <---
76142024-02-07T18:57:46.747Z 29 580-599 60 60 60 96 96 96 F F F
76152024-02-07T18:57:46.747ZMax gen: 68, Max flush: 110
76162024-02-07T18:57:46.747ZError: Difference in extent metadata found!
76172024-02-07T18:57:46.747ZA Difference in extent metadata is expected here
76182024-02-07T18:57:46.747ZOn loop 34
76192024-02-07T18:57:46.748Z
76202024-02-07T18:57:47.170ZVerifying data now
76212024-02-07T18:57:52.369ZNo change after verify
76222024-02-07T18:57:52.369ZLoop: 34 Downstairs dump after verify (and repair):
76232024-02-07T18:57:52.386ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76242024-02-07T18:57:52.386Z 0 000-019 67 67 67 107 107 107 F F F
76252024-02-07T18:57:52.386Z 1 020-039 57 57 57 94 94 94 F F F
76262024-02-07T18:57:52.386Z 2 040-059 60 60 60 96 96 96 F F F
76272024-02-07T18:57:52.386Z 3 060-079 64 64 64 104 104 104 F F F
76282024-02-07T18:57:52.386Z 4 080-099 66 66 66 106 106 106 F F F
76292024-02-07T18:57:52.386Z 5 100-119 66 66 66 106 106 106 F F F
76302024-02-07T18:57:52.390Z 6 120-139 69 69 69 111 111 111 F F F
76312024-02-07T18:57:52.390Z 7 140-159 68 68 68 110 110 110 F F F
76322024-02-07T18:57:52.390Z 8 160-179 68 68 68 110 110 110 F F F
76332024-02-07T18:57:52.390Z 9 180-199 57 57 57 94 94 94 F F F
76342024-02-07T18:57:52.390Z 10 200-219 68 68 68 110 110 110 F F F
76352024-02-07T18:57:52.390Z 11 220-239 68 68 68 110 110 110 F F F
76362024-02-07T18:57:52.390Z 12 240-259 68 68 68 108 108 108 F F F
76372024-02-07T18:57:52.390Z 13 260-279 65 65 65 105 105 105 F F F
76382024-02-07T18:57:52.390Z 14 280-299 67 67 67 107 107 107 F F F
76392024-02-07T18:57:52.390Z 15 300-319 67 67 67 107 107 107 F F F
76402024-02-07T18:57:52.390Z 16 320-339 67 67 67 107 107 107 F F F
76412024-02-07T18:57:52.390Z 17 340-359 65 65 65 105 105 105 F F F
76422024-02-07T18:57:52.390Z 18 360-379 63 63 63 100 100 100 F F F
76432024-02-07T18:57:52.390Z 19 380-399 60 60 60 96 96 96 F F F
76442024-02-07T18:57:52.391Z 20 400-419 69 69 69 111 111 111 F F F
76452024-02-07T18:57:52.391Z 21 420-439 68 68 68 109 109 109 F F F
76462024-02-07T18:57:52.391Z 22 440-459 68 68 68 108 108 108 F F F
76472024-02-07T18:57:52.391Z 23 460-479 68 68 68 108 108 108 F F F
76482024-02-07T18:57:52.391Z 24 480-499 68 68 68 108 108 108 F F F
76492024-02-07T18:57:52.391Z 25 500-519 67 67 67 107 107 107 F F F
76502024-02-07T18:57:52.391Z 26 520-539 67 67 67 107 107 107 F F F
76512024-02-07T18:57:52.391Z 27 540-559 68 68 68 109 109 109 F F F
76522024-02-07T18:57:52.391Z 28 560-579 68 68 68 109 109 109 F F F
76532024-02-07T18:57:52.391Z 29 580-599 60 60 60 96 96 96 F F F
76542024-02-07T18:57:52.391ZMax gen: 69, Max flush: 111
76552024-02-07T18:57:52.391Z
76562024-02-07T18:57:52.391ZBegin loop 35
76572024-02-07T18:57:52.391ZDownstairs to restart: 0
76582024-02-07T18:57:52.406Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652031436Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415}
76592024-02-07T18:57:52.406Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652322626Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415}
76602024-02-07T18:57:52.406Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652349223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415}
76612024-02-07T18:57:52.414Z{"msg":"Crucible 7ed2c740-8bfe-4f19-a0f1-100d098ec743 has session id: 7c78add5-aebb-47a4-b9ad-347011ca8959","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652389777Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76622024-02-07T18:57:52.414Z{"msg":"Upstairs opts: Upstairs UUID: 7ed2c740-8bfe-4f19-a0f1-100d098ec743, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
76632024-02-07T18:57:52.414Z2024-02-07T18:57:51.652435331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76642024-02-07T18:57:52.414Z{"msg":"Crucible stats registered with UUID: 7ed2c740-8bfe-4f19-a0f1-100d098ec743","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652497423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76652024-02-07T18:57:52.414Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652542927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415}
76662024-02-07T18:57:52.414Z{"msg":"7ed2c740-8bfe-4f19-a0f1-100d098ec743 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652711304Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76672024-02-07T18:57:52.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.65273835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76682024-02-07T18:57:52.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652778315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76692024-02-07T18:57:52.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652803861Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76702024-02-07T18:57:52.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652829088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76712024-02-07T18:57:52.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652870572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76722024-02-07T18:57:52.415Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652907827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76732024-02-07T18:57:52.415Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652935694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76742024-02-07T18:57:52.415Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.65296194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76752024-02-07T18:57:52.415Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.652994346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76762024-02-07T18:57:52.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653027161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76772024-02-07T18:57:52.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653050448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76782024-02-07T18:57:52.415Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:51.65318431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76792024-02-07T18:57:52.415Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:51.653243972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76802024-02-07T18:57:52.415Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:51.653395111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76812024-02-07T18:57:52.415Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653424657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76822024-02-07T18:57:52.415Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653440555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76832024-02-07T18:57:52.415Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653493438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76842024-02-07T18:57:52.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653832502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76852024-02-07T18:57:52.415Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653867667Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76862024-02-07T18:57:52.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653882985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76872024-02-07T18:57:52.415Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.653896713Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76882024-02-07T18:57:52.415Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.654506521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76892024-02-07T18:57:52.415Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.654542516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76902024-02-07T18:57:52.415Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.655122338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76912024-02-07T18:57:52.415Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.655158723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76922024-02-07T18:57:52.415Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.655191598Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76932024-02-07T18:57:52.415Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.655212626Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76942024-02-07T18:57:52.415Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.655227004Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76952024-02-07T18:57:52.416Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:51.655240832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76962024-02-07T18:57:53.410Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.654956869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76972024-02-07T18:57:53.410Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.655051056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76982024-02-07T18:57:53.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.655855448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
76992024-02-07T18:57:53.430Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.655890583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77002024-02-07T18:57:53.430Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.656459156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77012024-02-07T18:57:53.430Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657035238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77022024-02-07T18:57:53.431Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657079002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77032024-02-07T18:57:53.431Z{"msg":"[0]R flush_numbers[0..12]: [107, 94, 96, 104, 106, 106, 111, 110, 110, 94, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657107818Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77042024-02-07T18:57:53.431Z{"msg":"[0]R generation[0..12]: [67, 57, 60, 64, 66, 66, 69, 68, 68, 57, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657142923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77052024-02-07T18:57:53.431Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657176299Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77062024-02-07T18:57:53.431Z{"msg":"[1]R flush_numbers[0..12]: [107, 94, 96, 104, 106, 106, 111, 110, 110, 94, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657213014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
77072024-02-07T18:57:53.431Z7c78add5-aebb-47a4-b9ad-347011ca8959"}
77082024-02-07T18:57:53.431Z{"msg":"[1]R generation[0..12]: [67, 57, 60, 64, 66, 66, 69, 68, 68, 57, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657248739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77092024-02-07T18:57:53.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657281535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77102024-02-07T18:57:53.431Z{"msg":"[2]R flush_numbers[0..12]: [107, 94, 96, 104, 106, 106, 111, 110, 110, 94, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.65731513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77112024-02-07T18:57:53.432Z{"msg":"[2]R generation[0..12]: [67, 57, 60, 64, 66, 66, 69, 68, 68, 57, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657348056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77122024-02-07T18:57:53.432Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657371923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77132024-02-07T18:57:53.432Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657402948Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77142024-02-07T18:57:53.432Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657421666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77152024-02-07T18:57:53.432Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657450092Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77162024-02-07T18:57:53.432Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657468369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77172024-02-07T18:57:53.432Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657497686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77182024-02-07T18:57:53.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657514413Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77192024-02-07T18:57:53.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657544059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77202024-02-07T18:57:53.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657563257Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77212024-02-07T18:57:53.432Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657594062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77222024-02-07T18:57:53.432Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.65761275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77232024-02-07T18:57:53.433Z{"msg":"7ed2c740-8bfe-4f19-a0f1-100d098ec743 is now active with session: 7c78add5-aebb-47a4-b9ad-347011ca8959","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657644226Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77242024-02-07T18:57:53.433Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657663303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415,"session_id":"7c78add5-aebb-47a4-b9ad-347011ca8959"}
77252024-02-07T18:57:53.433ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77262024-02-07T18:57:53.433Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:52.657693649Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415}
77272024-02-07T18:57:53.433ZLoading write count information from file "/tmp/test_repair_verify.data"
77282024-02-07T18:57:53.433ZRun Repair workload
77292024-02-07T18:57:53.433Z01/30 Write block 144 len 24576 data: 8 8 10 10 10 12
77302024-02-07T18:57:53.433Z02/30 Write block 496 len 12288 data: 6 4 4
77312024-02-07T18:57:53.433Z03/30 Read block 127 len 32768
77322024-02-07T18:57:53.433Z04/30 Write block 518 len 40960 data: 5 4 5 3 3 3 6 5 6 8
77332024-02-07T18:57:53.452Z05/30 Write block 192 len 32768 data: 7 6 6 5 6 6 8 5
77342024-02-07T18:57:53.452Z06/30 Read block 109 len 8192
77352024-02-07T18:57:53.452Z07/30 Read block 74 len 4096
77362024-02-07T18:57:53.452Z08/30 Read block 101 len 40960
77372024-02-07T18:57:53.452Z09/30 Read block 356 len 28672
77382024-02-07T18:57:53.452Z10/30 Write block 262 len 36864 data: 8 8 8 7 5 4 4 4 3
77392024-02-07T18:57:53.452Z11/30 Read block 144 len 24576
77402024-02-07T18:57:53.452Z12/30 Write block 71 len 28672 data: 10 8 7 4 4 4 5
77412024-02-07T18:57:53.452Z13/30 Read block 564 len 32768
77422024-02-07T18:57:53.452Z14/30 Read block 363 len 16384
77432024-02-07T18:57:53.452Z15/30 Read block 171 len 24576
77442024-02-07T18:57:53.452Z16/30 Write block 158 len 12288 data: 5 5 5
77452024-02-07T18:57:53.452Z17/30 Read block 166 len 32768
77462024-02-07T18:57:53.452Z18/30 Flush
77472024-02-07T18:57:53.452Z19/30 Write block 445 len 4096 data: 9
77482024-02-07T18:57:53.453Z20/30 Write block 475 len 12288 data: 4 5 4
77492024-02-07T18:57:53.453Z21/30 Write block 78 len 4096 data: 7
77502024-02-07T18:57:53.453Z22/30 Write block 231 len 36864 data: 7 10 10 11 15 13 12 11 11
77512024-02-07T18:57:53.453Z23/30 Write block 503 len 12288 data: 7 7 8
77522024-02-07T18:57:53.453Z24/30 Read block 570 len 24576
77532024-02-07T18:57:53.453Z25/30 Write block 140 len 20480 data: 4 8 8 9 9
77542024-02-07T18:57:53.453Z26/30 Write block 200 len 12288 data: 4 3 5
77552024-02-07T18:57:53.453Z27/30 Read block 45 len 24576
77562024-02-07T18:57:53.453Z28/30 Write block 228 len 24576 data: 4 6 6 8 11 11
77572024-02-07T18:57:53.453Z29/30 Read block 312 len 28672
77582024-02-07T18:57:53.460Z30/30 Read block 344 len 24576
77592024-02-07T18:57:53.460Z----------------------------------------------------------------
77602024-02-07T18:57:53.460Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
77612024-02-07T18:57:53.460ZStates: Active Active Active
77622024-02-07T18:57:53.460ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
77632024-02-07T18:57:53.460Z 1 Acked 1000 Write 6 Sent Done Done false
77642024-02-07T18:57:53.460Z 2 Acked 1001 Write 3 Sent Done Done false
77652024-02-07T18:57:53.460Z 3 Acked 1002 Read 8 Sent Done Done false
77662024-02-07T18:57:53.460Z 4 Acked 1003 Write 10 Sent Done Done false
77672024-02-07T18:57:53.460Z 5 Acked 1004 Write 8 Sent Done Done false
77682024-02-07T18:57:53.460Z 6 Acked 1005 Read 2 Sent Done Done false
77692024-02-07T18:57:53.460Z 7 Acked 1006 Read 1 Sent Done Done false
77702024-02-07T18:57:53.460Z 8 Acked 1007 Read 10 Sent Done Done false
77712024-02-07T18:57:53.460Z 9 Acked 1008 Read 7 Sent Done Done false
77722024-02-07T18:57:53.460Z 10 Acked 1009 Write 9 Sent Done Done false
77732024-02-07T18:57:53.460Z 11 Acked 1010 Read 6 Sent Done Done false
77742024-02-07T18:57:53.460Z 12 Acked 1011 Write 7 Sent Done Done false
77752024-02-07T18:57:53.460Z 13 Acked 1012 Read 8 Sent Done Done false
77762024-02-07T18:57:53.461Z 14 Acked 1013 Read 4 Sent Done Done false
77772024-02-07T18:57:53.461Z 15 Acked 1014 Read 6 Sent Done Done false
77782024-02-07T18:57:53.461Z 16 Acked 1015 Write 3 Sent Done Done false
77792024-02-07T18:57:53.461Z 17 Acked 1016 Read 8 Sent Done Done false
77802024-02-07T18:57:53.461Z 18 Acked 1017 Flush 0 Sent Done Done false
77812024-02-07T18:57:53.461Z 19 Acked 1018 Write 1 Sent Done Done false
77822024-02-07T18:57:53.461Z 20 Acked 1019 Write 3 Sent Done Done false
77832024-02-07T18:57:53.461Z 21 Acked 1020 Write 1 Sent Done Done false
77842024-02-07T18:57:53.461Z 22 Acked 1021 Write 9 Sent Done Done false
77852024-02-07T18:57:53.461Z 23 Acked 1022 Write 3 Sent Done Done false
77862024-02-07T18:57:53.461Z 24 Acked 1023 Read 6 Sent Done Done false
77872024-02-07T18:57:53.461Z 25 Acked 1024 Write 5 Sent Done Done false
77882024-02-07T18:57:53.461Z 26 Acked 1025 Write 3 Sent Done Done false
77892024-02-07T18:57:53.461Z 27 Acked 1026 Read 6 Sent Done Done false
77902024-02-07T18:57:53.461Z 28 Acked 1027 Write 6 Sent Done Done false
77912024-02-07T18:57:53.461Z 29 Acked 1028 Read 7 Sent Done Done false
77922024-02-07T18:57:53.461Z 30 Acked 1029 Read 6 Sent Done Done false
77932024-02-07T18:57:53.461Z STATES DS:0 DS:1 DS:2 TOTAL
77942024-02-07T18:57:53.461Z New 0 0 0 0
77952024-02-07T18:57:53.461Z Sent 30 0 0 30
77962024-02-07T18:57:53.461Z Done 0 30 30 60
77972024-02-07T18:57:53.461Z Skipped 0 0 0 0
77982024-02-07T18:57:53.461Z Error 0 0 0 0
77992024-02-07T18:57:53.461ZLast Flush: 0 1017 1017
78002024-02-07T18:57:53.461ZDownstairs last five completed:
78012024-02-07T18:57:53.462ZUpstairs last five completed: 30 29 28 27 26
78022024-02-07T18:57:53.462Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:52.687397382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1415}
78032024-02-07T18:57:53.462ZWrote out file "/tmp/test_repair_verify.data"
78042024-02-07T18:57:53.462Z
78052024-02-07T18:57:53.462ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78062024-02-07T18:57:53.467ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78072024-02-07T18:57:53.467Z 0 000-019 67 67 67 107 107 107 F F F
78082024-02-07T18:57:53.467Z 1 020-039 57 57 57 94 94 94 F F F
78092024-02-07T18:57:53.467Z 2 040-059 60 60 60 96 96 96 F F F
78102024-02-07T18:57:53.467Z 3 060-079 64 70 70 104 112 112 F T T <---
78112024-02-07T18:57:53.467Z 4 080-099 66 66 66 106 106 106 F F F
78122024-02-07T18:57:53.467Z 5 100-119 66 66 66 106 106 106 F F F
78132024-02-07T18:57:53.468Z 6 120-139 69 69 69 111 111 111 F F F
78142024-02-07T18:57:53.468Z 7 140-159 68 70 70 110 112 112 F T T <---
78152024-02-07T18:57:53.471Z 8 160-179 68 70 70 110 112 112 F F F <---
78162024-02-07T18:57:53.471Z 9 180-199 57 70 70 94 112 112 F F F <---
78172024-02-07T18:57:53.471Z 10 200-219 68 68 68 110 110 110 F T T <---
78182024-02-07T18:57:53.471Z 11 220-239 68 68 68 110 110 110 F T T <---
78192024-02-07T18:57:53.471Z 12 240-259 68 68 68 108 108 108 F F F
78202024-02-07T18:57:53.471Z 13 260-279 65 70 70 105 112 112 F F F <---
78212024-02-07T18:57:53.471Z 14 280-299 67 67 67 107 107 107 F F F
78222024-02-07T18:57:53.471Z 15 300-319 67 67 67 107 107 107 F F F
78232024-02-07T18:57:53.471Z 16 320-339 67 67 67 107 107 107 F F F
78242024-02-07T18:57:53.471Z 17 340-359 65 65 65 105 105 105 F F F
78252024-02-07T18:57:53.471Z 18 360-379 63 63 63 100 100 100 F F F
78262024-02-07T18:57:53.471Z 19 380-399 60 60 60 96 96 96 F F F
78272024-02-07T18:57:53.472Z 20 400-419 69 69 69 111 111 111 F F F
78282024-02-07T18:57:53.472Z 21 420-439 68 68 68 109 109 109 F F F
78292024-02-07T18:57:53.472Z 22 440-459 68 68 68 108 108 108 F T T <---
78302024-02-07T18:57:53.472Z 23 460-479 68 68 68 108 108 108 F T T <---
78312024-02-07T18:57:53.472Z 24 480-499 68 70 70 108 112 112 F F F <---
78322024-02-07T18:57:53.472Z 25 500-519 67 70 70 107 112 112 F T T <---
78332024-02-07T18:57:53.472Z 26 520-539 67 70 70 107 112 112 F F F <---
78342024-02-07T18:57:53.472Z 27 540-559 68 68 68 109 109 109 F F F
78352024-02-07T18:57:53.472Z 28 560-579 68 68 68 109 109 109 F F F
78362024-02-07T18:57:53.472Z 29 580-599 60 60 60 96 96 96 F F F
78372024-02-07T18:57:53.472ZMax gen: 70, Max flush: 112
78382024-02-07T18:57:53.472ZError: Difference in extent metadata found!
78392024-02-07T18:57:53.472ZA Difference in extent metadata is expected here
78402024-02-07T18:57:53.472ZOn loop 35
78412024-02-07T18:57:53.472Z
78422024-02-07T18:57:53.475ZVerifying data now
78432024-02-07T18:57:58.480ZNo change after verify
78442024-02-07T18:57:58.480ZLoop: 35 Downstairs dump after verify (and repair):
78452024-02-07T18:57:58.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78462024-02-07T18:57:58.503Z 0 000-019 67 67 67 107 107 107 F F F
78472024-02-07T18:57:58.503Z 1 020-039 57 57 57 94 94 94 F F F
78482024-02-07T18:57:58.503Z 2 040-059 60 60 60 96 96 96 F F F
78492024-02-07T18:57:58.503Z 3 060-079 71 71 71 113 113 113 F F F
78502024-02-07T18:57:58.503Z 4 080-099 66 66 66 106 106 106 F F F
78512024-02-07T18:57:58.504Z 5 100-119 66 66 66 106 106 106 F F F
78522024-02-07T18:57:58.504Z 6 120-139 69 69 69 111 111 111 F F F
78532024-02-07T18:57:58.504Z 7 140-159 71 71 71 113 113 113 F F F
78542024-02-07T18:57:58.504Z 8 160-179 70 70 70 112 112 112 F F F
78552024-02-07T18:57:58.504Z 9 180-199 70 70 70 112 112 112 F F F
78562024-02-07T18:57:58.504Z 10 200-219 71 71 71 113 113 113 F F F
78572024-02-07T18:57:58.504Z 11 220-239 71 71 71 113 113 113 F F F
78582024-02-07T18:57:58.504Z 12 240-259 68 68 68 108 108 108 F F F
78592024-02-07T18:57:58.504Z 13 260-279 70 70 70 112 112 112 F F F
78602024-02-07T18:57:58.504Z 14 280-299 67 67 67 107 107 107 F F F
78612024-02-07T18:57:58.504Z 15 300-319 67 67 67 107 107 107 F F F
78622024-02-07T18:57:58.505Z 16 320-339 67 67 67 107 107 107 F F F
78632024-02-07T18:57:58.505Z 17 340-359 65 65 65 105 105 105 F F F
78642024-02-07T18:57:58.505Z 18 360-379 63 63 63 100 100 100 F F F
78652024-02-07T18:57:58.505Z 19 380-399 60 60 60 96 96 96 F F F
78662024-02-07T18:57:58.505Z 20 400-419 69 69 69 111 111 111 F F F
78672024-02-07T18:57:58.505Z 21 420-439 68 68 68 109 109 109 F F F
78682024-02-07T18:57:58.505Z 22 440-459 71 71 71 113 113 113 F F F
78692024-02-07T18:57:58.505Z 23 460-479 71 71 71 113 113 113 F F F
78702024-02-07T18:57:58.505Z 24 480-499 70 70 70 112 112 112 F F F
78712024-02-07T18:57:58.505Z 25 500-519 71 71 71 113 113 113 F F F
78722024-02-07T18:57:58.505Z 26 520-539 70 70 70 112 112 112 F F F
78732024-02-07T18:57:58.505Z 27 540-559 68 68 68 109 109 109 F F F
78742024-02-07T18:57:58.506Z 28 560-579 68 68 68 109 109 109 F F F
78752024-02-07T18:57:58.506Z 29 580-599 60 60 60 96 96 96 F F F
78762024-02-07T18:57:58.506ZMax gen: 71, Max flush: 113
78772024-02-07T18:57:58.506Z
78782024-02-07T18:57:58.506ZBegin loop 36
78792024-02-07T18:57:58.506ZDownstairs to restart: 2
78802024-02-07T18:57:58.506Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359056448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423}
78812024-02-07T18:57:58.506ZCrucible runtime is spawned
78822024-02-07T18:57:58.506Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359455254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423}
78832024-02-07T18:57:58.506Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359486349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423}
78842024-02-07T18:57:58.506Z{"msg":"Crucible a4bdf669-6ac8-4e54-b201-8a73af9c2a7e has session id: 418f842b-e4a9-440c-afeb-d74f73401ebf","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359517375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78852024-02-07T18:57:58.506Z{"msg":"Upstairs opts: Upstairs UUID: a4bdf669-6ac8-4e54-b201-8a73af9c2a7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359546641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78862024-02-07T18:57:58.506Z{"msg":"Crucible stats registered with UUID: a4bdf669-6ac8-4e54-b201-8a73af9c2a7e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359575577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78872024-02-07T18:57:58.506Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359592285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423}
78882024-02-07T18:57:58.506Z{"msg":"a4bdf669-6ac8-4e54-b201-8a73af9c2a7e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359621761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78892024-02-07T18:57:58.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359676474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78902024-02-07T18:57:58.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359727097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78912024-02-07T18:57:58.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359773391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78922024-02-07T18:57:58.507Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359805486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78932024-02-07T18:57:58.507Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359833542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78942024-02-07T18:57:58.507Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359871207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78952024-02-07T18:57:58.507Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359889425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78962024-02-07T18:57:58.507Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359922901Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78972024-02-07T18:57:58.507Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.359941158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78982024-02-07T18:57:58.507Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:57.359974023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
78992024-02-07T18:57:58.507Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:57.360018188Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79002024-02-07T18:57:58.507Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:57.360048723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79012024-02-07T18:57:58.507Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.36007125Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79022024-02-07T18:57:58.507Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360095867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79032024-02-07T18:57:58.507Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360119384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79042024-02-07T18:57:58.507Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360144711Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79052024-02-07T18:57:58.507Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360169407Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79062024-02-07T18:57:58.507Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360757588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79072024-02-07T18:57:58.507Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360795562Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79082024-02-07T18:57:58.507Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.36081273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79092024-02-07T18:57:58.507Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.360826668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79102024-02-07T18:57:58.507Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.361392322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79112024-02-07T18:57:58.507Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.361495558Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79122024-02-07T18:57:58.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.361972003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79132024-02-07T18:57:58.508Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.362018767Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79142024-02-07T18:57:58.508Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.362045993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79152024-02-07T18:57:58.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.362073279Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79162024-02-07T18:57:58.508Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.362098306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79172024-02-07T18:57:58.508Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:57.362126012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79182024-02-07T18:57:59.117Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.361748473Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79192024-02-07T18:57:59.117Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.361863508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79202024-02-07T18:57:59.124Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.362741709Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79212024-02-07T18:57:59.124Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.362778514Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79222024-02-07T18:57:59.124Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.363331059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79232024-02-07T18:57:59.124Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.363901982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79242024-02-07T18:57:59.124Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.363944866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79252024-02-07T18:57:59.124Z{"msg":"[0]R flush_numbers[0..12]: [107, 94, 96, 113, 106, 106, 111, 113, 112, 112, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.363973483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79262024-02-07T18:57:59.124Z{"msg":"[0]R generation[0..12]: [67, 57, 60, 71, 66, 66, 69, 71, 70, 70, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364057701Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79272024-02-07T18:57:59.124Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364078178Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79282024-02-07T18:57:59.124Z{"msg":"[1]R flush_numbers[0..12]: [107, 94, 96, 113, 106, 106, 111, 113, 112, 112, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364104845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79292024-02-07T18:57:59.124Z{"msg":"[1]R generation[0..12]: [67, 57, 60, 71, 66, 66, 69, 71, 70, 70, 71, 71]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
79302024-02-07T18:57:59.124Z,"time":"2024-02-07T18:57:58.364124662Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79312024-02-07T18:57:59.124Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364154698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79322024-02-07T18:57:59.124Z{"msg":"[2]R flush_numbers[0..12]: [107, 94, 96, 113, 106, 106, 111, 113, 112, 112, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364180844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79332024-02-07T18:57:59.125Z{"msg":"[2]R generation[0..12]: [67, 57, 60, 71, 66, 66, 69, 71, 70, 70, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364199462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79342024-02-07T18:57:59.125Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364232097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79352024-02-07T18:57:59.125Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364249795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79362024-02-07T18:57:59.125Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364268352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79372024-02-07T18:57:59.125Z{"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.36428482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79382024-02-07T18:57:59.125Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364310607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79392024-02-07T18:57:59.125Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364327684Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79402024-02-07T18:57:59.125Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364351431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79412024-02-07T18:57:59.125Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364368739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79422024-02-07T18:57:59.125Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364393746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79432024-02-07T18:57:59.125Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364410583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79442024-02-07T18:57:59.125Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.36443399Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79452024-02-07T18:57:59.125Z{"msg":"a4bdf669-6ac8-4e54-b201-8a73af9c2a7e is now active with session: 418f842b-e4a9-440c-afeb-d74f73401ebf","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364450768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79462024-02-07T18:57:59.125Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364473975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423,"session_id":"418f842b-e4a9-440c-afeb-d74f73401ebf"}
79472024-02-07T18:57:59.125Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:57:58.364490222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423}
79482024-02-07T18:57:59.125ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
79492024-02-07T18:57:59.125ZLoading write count information from file "/tmp/test_repair_verify.data"
79502024-02-07T18:57:59.125ZRun Repair workload
79512024-02-07T18:57:59.125Z01/30 Write block 483 len 8192 data: 4 3
79522024-02-07T18:57:59.130Z02/30 Read block 388 len 12288
79532024-02-07T18:57:59.130Z03/30 Read block 332 len 28672
79542024-02-07T18:57:59.130Z04/30 Read block 421 len 40960
79552024-02-07T18:57:59.130Z05/30 Write block 508 len 12288 data: 6 6 8
79562024-02-07T18:57:59.130Z06/30 Read block 203 len 12288
79572024-02-07T18:57:59.130Z07/30 Read block 56 len 28672
79582024-02-07T18:57:59.130Z08/30 Write block 202 len 4096 data: 6
79592024-02-07T18:57:59.130Z09/30 Flush
79602024-02-07T18:57:59.133Z10/30 Write block 430 len 32768 data: 7 7 5 3 3 4 6 8
79612024-02-07T18:57:59.134Z11/30 Write block 549 len 28672 data: 8 7 6 6 7 5 6
79622024-02-07T18:57:59.134Z12/30 Write block 458 len 36864 data: 7 6 5 5 5 6 6 6 6
79632024-02-07T18:57:59.134Z13/30 Read block 226 len 16384
79642024-02-07T18:57:59.137Z14/30 Read block 6 len 8192
79652024-02-07T18:57:59.137Z15/30 Read block 297 len 20480
79662024-02-07T18:57:59.137Z16/30 Flush
79672024-02-07T18:57:59.141Z17/30 Write block 5 len 20480 data: 6 5 8 7 8
79682024-02-07T18:57:59.141Z18/30 Write block 470 len 28672 data: 4 4 3 3 3 5 6
79692024-02-07T18:57:59.141Z19/30 Read block 512 len 28672
79702024-02-07T18:57:59.141Z20/30 Write block 252 len 12288 data: 5 4 4
79712024-02-07T18:57:59.145Z21/30 Write block 229 len 4096 data: 7
79722024-02-07T18:57:59.145Z22/30 Write block 366 len 32768 data: 7 5 6 5 5 4 3 3
79732024-02-07T18:57:59.145Z23/30 Write block 30 len 8192 data: 10 10
79742024-02-07T18:57:59.145Z24/30 Write block 103 len 24576 data: 4 7 8 7 7 7
79752024-02-07T18:57:59.145Z25/30 Write block 131 len 28672 data: 5 5 5 5 3 3 4
79762024-02-07T18:57:59.145Z26/30 Flush
79772024-02-07T18:57:59.150Z27/30 Write block 541 len 4096 data: 11
79782024-02-07T18:57:59.150Z28/30 Read block 311 len 16384
79792024-02-07T18:57:59.150Z29/30 Write block 487 len 24576 data: 3 5 4 5 5 5
79802024-02-07T18:57:59.154Z30/30 Read block 31 len 8192
79812024-02-07T18:57:59.154Z----------------------------------------------------------------
79822024-02-07T18:57:59.154Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
79832024-02-07T18:57:59.154ZStates: Active Active Active
79842024-02-07T18:57:59.154ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
79852024-02-07T18:57:59.154Z 1 Acked 1000 Write 2 Done Done Sent false
79862024-02-07T18:57:59.155Z 2 Acked 1001 Read 3 Done Done Sent false
79872024-02-07T18:57:59.155Z 3 Acked 1002 Read 7 Done Done Sent false
79882024-02-07T18:57:59.155Z 4 Acked 1003 Read 10 Done Done Sent false
79892024-02-07T18:57:59.155Z 5 Acked 1004 Write 3 Done Done Sent false
79902024-02-07T18:57:59.155Z 6 Acked 1005 Read 3 Done Done Sent false
79912024-02-07T18:57:59.155Z 7 Acked 1006 Read 7 Done Done Sent false
79922024-02-07T18:57:59.155Z 8 Acked 1007 Write 1 Done Done Sent false
79932024-02-07T18:57:59.155Z 9 Acked 1008 Flush 0 Done Done Sent false
79942024-02-07T18:57:59.155Z 10 Acked 1009 Write 8 Done Done Sent false
79952024-02-07T18:57:59.155Z 11 Acked 1010 Write 7 Done Done Sent false
79962024-02-07T18:57:59.155Z 12 Acked 1011 Write 9 Done Done Sent false
79972024-02-07T18:57:59.155Z 13 Acked 1012 Read 4 Done Done Sent false
79982024-02-07T18:57:59.155Z 14 Acked 1013 Read 2 Done Done Sent false
79992024-02-07T18:57:59.156Z 15 Acked 1014 Read 5 Done Done Sent false
80002024-02-07T18:57:59.156Z 16 Acked 1015 Flush 0 Done Done Sent false
80012024-02-07T18:57:59.156Z 17 Acked 1016 Write 5 Done Done Sent false
80022024-02-07T18:57:59.156Z 18 Acked 1017 Write 7 Done Done Sent false
80032024-02-07T18:57:59.156Z 19 Acked 1018 Read 7 Done Done Sent false
80042024-02-07T18:57:59.156Z 20 Acked 1019 Write 3 Done Done Sent false
80052024-02-07T18:57:59.156Z 21 Acked 1020 Write 1 Done Done Sent false
80062024-02-07T18:57:59.156Z 22 Acked 1021 Write 8 Done Done Sent false
80072024-02-07T18:57:59.156Z 23 Acked 1022 Write 2 Done Done Sent false
80082024-02-07T18:57:59.156Z 24 Acked 1023 Write 6 Done Done Sent false
80092024-02-07T18:57:59.156Z 25 Acked 1024 Write 7 Done Done Sent false
80102024-02-07T18:57:59.156Z 26 Acked 1025 Flush 0 Done Done Sent false
80112024-02-07T18:57:59.156Z 27 Acked 1026 Write 1 Done Done Sent false
80122024-02-07T18:57:59.157Z 28 Acked 1027 Read 4 Done Done Sent false
80132024-02-07T18:57:59.157Z 29 Acked 1028 Write 6 Done Done Sent false
80142024-02-07T18:57:59.157Z 30 Acked 1029 Read 2 Sent Done Sent false
80152024-02-07T18:57:59.157Z STATES DS:0 DS:1 DS:2 TOTAL
80162024-02-07T18:57:59.157Z New 0 0 0 0
80172024-02-07T18:57:59.157Z Sent 1 0 30 31
80182024-02-07T18:57:59.157Z Done 29 30 0 59
80192024-02-07T18:57:59.157Z Skipped 0 0 0 0
80202024-02-07T18:57:59.157Z Error 0 0 0 0
80212024-02-07T18:57:59.157ZLast Flush: 1025 1025 0
80222024-02-07T18:57:59.157ZDownstairs last five completed:
80232024-02-07T18:57:59.157ZUpstairs last five completed: 30 29 28 27 26
80242024-02-07T18:57:59.157Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:57:58.39504223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1423}
80252024-02-07T18:57:59.158ZWrote out file "/tmp/test_repair_verify.data"
80262024-02-07T18:57:59.162Z
80272024-02-07T18:57:59.162ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80282024-02-07T18:57:59.175ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80292024-02-07T18:57:59.175Z 0 000-019 72 72 67 116 116 107 F F F <---
80302024-02-07T18:57:59.175Z 1 020-039 72 72 57 116 116 94 F F F <---
80312024-02-07T18:57:59.175Z 2 040-059 60 60 60 96 96 96 F F F
80322024-02-07T18:57:59.175Z 3 060-079 71 71 71 113 113 113 F F F
80332024-02-07T18:57:59.175Z 4 080-099 66 66 66 106 106 106 F F F
80342024-02-07T18:57:59.175Z 5 100-119 72 72 66 116 116 106 F F F <---
80352024-02-07T18:57:59.175Z 6 120-139 72 72 69 116 116 111 F F F <---
80362024-02-07T18:57:59.176Z 7 140-159 71 71 71 113 113 113 F F F
80372024-02-07T18:57:59.179Z 8 160-179 70 70 70 112 112 112 F F F
80382024-02-07T18:57:59.179Z 9 180-199 70 70 70 112 112 112 F F F
80392024-02-07T18:57:59.179Z 10 200-219 72 72 71 114 114 113 F F F <---
80402024-02-07T18:57:59.179Z 11 220-239 72 72 71 116 116 113 F F F <---
80412024-02-07T18:57:59.179Z 12 240-259 72 72 68 116 116 108 F F F <---
80422024-02-07T18:57:59.179Z 13 260-279 70 70 70 112 112 112 F F F
80432024-02-07T18:57:59.179Z 14 280-299 67 67 67 107 107 107 F F F
80442024-02-07T18:57:59.179Z 15 300-319 67 67 67 107 107 107 F F F
80452024-02-07T18:57:59.179Z 16 320-339 67 67 67 107 107 107 F F F
80462024-02-07T18:57:59.179Z 17 340-359 65 65 65 105 105 105 F F F
80472024-02-07T18:57:59.179Z 18 360-379 72 72 63 116 116 100 F F F <---
80482024-02-07T18:57:59.180Z 19 380-399 60 60 60 96 96 96 F F F
80492024-02-07T18:57:59.180Z 20 400-419 69 69 69 111 111 111 F F F
80502024-02-07T18:57:59.180Z 21 420-439 72 72 68 115 115 109 F F F <---
80512024-02-07T18:57:59.180Z 22 440-459 72 72 71 115 115 113 F F F <---
80522024-02-07T18:57:59.180Z 23 460-479 72 72 71 116 116 113 F F F <---
80532024-02-07T18:57:59.180Z 24 480-499 72 72 70 114 114 112 T T F <---
80542024-02-07T18:57:59.180Z 25 500-519 72 72 71 114 114 113 F F F <---
80552024-02-07T18:57:59.180Z 26 520-539 70 70 70 112 112 112 F F F
80562024-02-07T18:57:59.180Z 27 540-559 72 72 68 115 115 109 T T F <---
80572024-02-07T18:57:59.180Z 28 560-579 68 68 68 109 109 109 F F F
80582024-02-07T18:57:59.180Z 29 580-599 60 60 60 96 96 96 F F F
80592024-02-07T18:57:59.180ZMax gen: 72, Max flush: 116
80602024-02-07T18:57:59.180ZError: Difference in extent metadata found!
80612024-02-07T18:57:59.180ZA Difference in extent metadata is expected here
80622024-02-07T18:57:59.180ZOn loop 36
80632024-02-07T18:57:59.180Z
80642024-02-07T18:57:59.184ZVerifying data now
80652024-02-07T18:58:14.125ZNo change after verify
80662024-02-07T18:58:14.125ZLoop: 36 Downstairs dump after verify (and repair):
80672024-02-07T18:58:14.298ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80682024-02-07T18:58:14.298Z 0 000-019 72 72 72 116 116 116 F F F
80692024-02-07T18:58:14.298Z 1 020-039 72 72 72 116 116 116 F F F
80702024-02-07T18:58:14.298Z 2 040-059 60 60 60 96 96 96 F F F
80712024-02-07T18:58:14.298Z 3 060-079 71 71 71 113 113 113 F F F
80722024-02-07T18:58:14.299Z 4 080-099 66 66 66 106 106 106 F F F
80732024-02-07T18:58:14.299Z 5 100-119 72 72 72 116 116 116 F F F
80742024-02-07T18:58:14.299Z 6 120-139 72 72 72 116 116 116 F F F
80752024-02-07T18:58:14.299Z 7 140-159 71 71 71 113 113 113 F F F
80762024-02-07T18:58:14.300Z 8 160-179 70 70 70 112 112 112 F F F
80772024-02-07T18:58:14.300Z 9 180-199 70 70 70 112 112 112 F F F
80782024-02-07T18:58:14.300Z 10 200-219 72 72 72 114 114 114 F F F
80792024-02-07T18:58:14.301Z 11 220-239 72 72 72 116 116 116 F F F
80802024-02-07T18:58:14.301Z 12 240-259 72 72 72 116 116 116 F F F
80812024-02-07T18:58:14.301Z 13 260-279 70 70 70 112 112 112 F F F
80822024-02-07T18:58:14.302Z 14 280-299 67 67 67 107 107 107 F F F
80832024-02-07T18:58:14.303Z 15 300-319 67 67 67 107 107 107 F F F
80842024-02-07T18:58:14.303Z 16 320-339 67 67 67 107 107 107 F F F
80852024-02-07T18:58:14.303Z 17 340-359 65 65 65 105 105 105 F F F
80862024-02-07T18:58:14.303Z 18 360-379 72 72 72 116 116 116 F F F
80872024-02-07T18:58:14.303Z 19 380-399 60 60 60 96 96 96 F F F
80882024-02-07T18:58:14.304Z 20 400-419 69 69 69 111 111 111 F F F
80892024-02-07T18:58:14.304Z 21 420-439 72 72 72 115 115 115 F F F
80902024-02-07T18:58:14.304Z 22 440-459 72 72 72 115 115 115 F F F
80912024-02-07T18:58:14.305Z 23 460-479 72 72 72 116 116 116 F F F
80922024-02-07T18:58:14.305Z 24 480-499 73 73 73 117 117 117 F F F
80932024-02-07T18:58:14.305Z 25 500-519 72 72 72 114 114 114 F F F
80942024-02-07T18:58:14.305Z 26 520-539 70 70 70 112 112 112 F F F
80952024-02-07T18:58:14.305Z 27 540-559 73 73 73 117 117 117 F F F
80962024-02-07T18:58:14.305Z 28 560-579 68 68 68 109 109 109 F F F
80972024-02-07T18:58:14.305Z 29 580-599 60 60 60 96 96 96 F F F
80982024-02-07T18:58:14.306ZMax gen: 73, Max flush: 117
80992024-02-07T18:58:14.306Z
81002024-02-07T18:58:14.306ZBegin loop 37
81012024-02-07T18:58:14.306ZDownstairs to restart: 1
81022024-02-07T18:58:14.306Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.154890448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431}
81032024-02-07T18:58:14.306Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155233222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431}
81042024-02-07T18:58:14.306Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name"Crucible runtime is spawned
81052024-02-07T18:58:14.306Z:"crucible","level":30,"time":"2024-02-07T18:58:03.155285424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431}
81062024-02-07T18:58:14.307Z{"msg":"Crucible 0b6cf502-edee-4731-82fe-fed785d9ec69 has session id: 0a2b9d55-76aa-454e-8725-b5a6b77e3af3","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155322599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81072024-02-07T18:58:14.307Z{"msg":"Upstairs opts: Upstairs UUID: 0b6cf502-edee-4731-82fe-fed785d9ec69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155351595Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81082024-02-07T18:58:14.307Z{"msg":"Crucible stats registered with UUID: 0b6cf502-edee-4731-82fe-fed785d9ec69","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155374672Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81092024-02-07T18:58:14.307Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155404278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431}
81102024-02-07T18:58:14.307Z{"msg":"0b6cf502-edee-4731-82fe-fed785d9ec69 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155488427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81112024-02-07T18:58:14.307Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155520173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81122024-02-07T18:58:14.307Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.15554Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81132024-02-07T18:58:14.307Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155581415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81142024-02-07T18:58:14.307Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.15561697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81152024-02-07T18:58:14.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155663683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81162024-02-07T18:58:14.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155735923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81172024-02-07T18:58:14.308Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155768569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81182024-02-07T18:58:14.308Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155902741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81192024-02-07T18:58:14.308Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155930807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81202024-02-07T18:58:14.308Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:03.155970142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81212024-02-07T18:58:14.308Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.155995428Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81222024-02-07T18:58:14.308Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:03.156034073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81232024-02-07T18:58:14.308Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.15606015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81242024-02-07T18:58:14.308Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:03.156098185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81252024-02-07T18:58:14.308Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.156137759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81262024-02-07T18:58:14.308Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.156161486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81272024-02-07T18:58:14.308Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.156192722Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81282024-02-07T18:58:14.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.156621004Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81292024-02-07T18:58:14.309Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.156666628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81302024-02-07T18:58:14.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.156702203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81312024-02-07T18:58:14.309Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.15672644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81322024-02-07T18:58:14.309Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157306951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81332024-02-07T18:58:14.309Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157355804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81342024-02-07T18:58:14.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157891122Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81352024-02-07T18:58:14.309Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157926937Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81362024-02-07T18:58:14.309Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157942655Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81372024-02-07T18:58:14.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157963332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81382024-02-07T18:58:14.310Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.15797759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81392024-02-07T18:58:14.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:03.157991229Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81402024-02-07T18:58:14.310Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.157733783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81412024-02-07T18:58:14.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.157817511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81422024-02-07T18:58:14.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.159104187Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81432024-02-07T18:58:14.310Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.159151211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81442024-02-07T18:58:14.310Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.159647334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81452024-02-07T18:58:14.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160200799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81462024-02-07T18:58:14.311Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160235474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81472024-02-07T18:58:14.311Z{"msg":"[0]R flush_numbers[0..12]: [116, 116, 96, 113, 106, 116, 116, 113, 112, 112, 114, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160254362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81482024-02-07T18:58:14.311Z{"msg":"[0]R generation[0..12]: [72, 72, 60, 71, 66, 72, 72, 71, 70, 70, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160272119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81492024-02-07T18:58:14.311Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160366466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81502024-02-07T18:58:14.311Z{"msg":"[1]R flush_numbers[0..12]: [116, 116, 96, 113, 106, 116, 116, 113, 112, 112, 114, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160400272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81512024-02-07T18:58:14.311Z{"msg":"[1]R generation[0..12]: [72, 72, 60, 71, 66, 72, 72, 71, 70, 70, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160419879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81522024-02-07T18:58:14.311Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160446236Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81532024-02-07T18:58:14.312Z{"msg":"[2]R flush_numbers[0..12]: [116, 116, 96, 113, 106, 116, 116, 113, 112, 112, 114, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160465443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81542024-02-07T18:58:14.312Z{"msg":"[2]R generation[0..12]: [72, 72, 60, 71, 66, 72, 72, 71, 70, 70, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81552024-02-07T18:58:14.312Z{"msg":"Wait for a query_work_queue command to finish before sending IO
81562024-02-07T18:58:14.312Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160513817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81572024-02-07T18:58:14.312Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160531784Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81582024-02-07T18:58:14.312Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160549812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81592024-02-07T18:58:14.313Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.16056691Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81602024-02-07T18:58:14.313Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160584357Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81612024-02-07T18:58:14.313Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160609574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81622024-02-07T18:58:14.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160628081Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81632024-02-07T18:58:14.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160646449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81642024-02-07T18:58:14.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160662866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81652024-02-07T18:58:14.313Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160702071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81662024-02-07T18:58:14.314Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160719539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81672024-02-07T18:58:14.382Z{"msg":"0b6cf502-edee-4731-82fe-fed785d9ec69 is now active with session: 0a2b9d55-76aa-454e-8725-b5a6b77e3af3","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160744495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81682024-02-07T18:58:14.382Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160760133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431,"session_id":"0a2b9d55-76aa-454e-8725-b5a6b77e3af3"}
81692024-02-07T18:58:14.383Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:04.160776651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431}
81702024-02-07T18:58:14.383ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
81712024-02-07T18:58:14.383ZLoading write count information from file "/tmp/test_repair_verify.data"
81722024-02-07T18:58:14.383ZRun Repair workload
81732024-02-07T18:58:14.383Z01/30 Write block 54 len 24576 data: 8 9 9 9 8 7
81742024-02-07T18:58:14.383Z02/30 Read block 357 len 8192
81752024-02-07T18:58:14.383Z03/30 Write block 90 len 28672 data: 6 6 7 7 9 8 8
81762024-02-07T18:58:14.383Z04/30 Read block 352 len 28672
81772024-02-07T18:58:14.383Z05/30 Read block 194 len 40960
81782024-02-07T18:58:14.383Z06/30 Read block 457 len 12288
81792024-02-07T18:58:14.383Z07/30 Read block 347 len 24576
81802024-02-07T18:58:14.383Z08/30 Read block 71 len 8192
81812024-02-07T18:58:14.383Z09/30 Flush
81822024-02-07T18:58:14.384Z10/30 Flush
81832024-02-07T18:58:14.384Z11/30 Write block 534 len 32768 data: 8 9 10 11 9 10 11 12
81842024-02-07T18:58:14.384Z12/30 Read block 357 len 28672
81852024-02-07T18:58:14.384Z13/30 Write block 348 len 28672 data: 8 9 9 8 5 4 6
81862024-02-07T18:58:14.384Z14/30 Read block 298 len 36864
81872024-02-07T18:58:14.384Z15/30 Write block 527 len 12288 data: 9 8 8
81882024-02-07T18:58:14.384Z16/30 Write block 213 len 4096 data: 7
81892024-02-07T18:58:14.384Z17/30 Write block 31 len 20480 data: 11 8 8 9 8
81902024-02-07T18:58:14.384Z18/30 Write block 168 len 24576 data: 10 9 8 8 8 8
81912024-02-07T18:58:14.384Z19/30 Read block 361 len 28672
81922024-02-07T18:58:14.384Z20/30 Write block 466 len 4096 data: 7
81932024-02-07T18:58:14.384Z21/30 Read block 390 len 16384
81942024-02-07T18:58:14.384Z22/30 Flush
81952024-02-07T18:58:14.385Z23/30 Flush
81962024-02-07T18:58:14.385Z24/30 Write block 76 len 8192 data: 5 6
81972024-02-07T18:58:14.385Z25/30 Read block 126 len 20480
81982024-02-07T18:58:14.385Z26/30 Write block 519 len 12288 data: 5 6 4
81992024-02-07T18:58:14.385Z27/30 Write block 191 len 36864 data: 7 8 7 7 6 7 7 9 6
82002024-02-07T18:58:14.385Z28/30 Write block 48 len 8192 data: 7 6
82012024-02-07T18:58:14.385Z29/30 Read block 296 len 28672
82022024-02-07T18:58:14.385Z30/30 Write block 496 len 40960 data: 7 5 5 4 4 3 6 8 8 9
82032024-02-07T18:58:14.385Z----------------------------------------------------------------
82042024-02-07T18:58:14.385Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
82052024-02-07T18:58:14.385ZStates: Active Active Active
82062024-02-07T18:58:14.385ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82072024-02-07T18:58:14.385Z 1 Acked 1000 Write 6 Done Done Done false
82082024-02-07T18:58:14.386Z 2 Acked 1001 Read 2 Done Done Done false
82092024-02-07T18:58:14.386Z 3 Acked 1002 Write 7 Done Sent Done false
82102024-02-07T18:58:14.386Z 4 Acked 1003 Read 7 Done Sent Done false
82112024-02-07T18:58:14.386Z 5 Acked 1004 Read 10 Done Sent Done false
82122024-02-07T18:58:14.386Z 6 Acked 1005 Read 3 Done Sent Done false
82132024-02-07T18:58:14.386Z 7 Acked 1006 Read 6 Done Sent Done false
82142024-02-07T18:58:14.386Z 8 Acked 1007 Read 2 Done Sent Done false
82152024-02-07T18:58:14.386Z 9 Acked 1008 Flush 0 Done Sent Done false
82162024-02-07T18:58:14.386Z 10 Acked 1009 Flush 0 Done Sent Done false
82172024-02-07T18:58:14.386Z 11 Acked 1010 Write 8 Done Sent Done false
82182024-02-07T18:58:14.386Z 12 Acked 1011 Read 7 Done Sent Done false
82192024-02-07T18:58:14.386Z 13 Acked 1012 Write 7 Done Sent Done false
82202024-02-07T18:58:14.386Z 14 Acked 1013 Read 9 Done Sent Done false
82212024-02-07T18:58:14.386Z 15 Acked 1014 Write 3 Done Sent Done false
82222024-02-07T18:58:14.386Z 16 Acked 1015 Write 1 Done Sent Done false
82232024-02-07T18:58:14.386Z 17 Acked 1016 Write 5 Done Sent Done false
82242024-02-07T18:58:14.387Z 18 Acked 1017 Write 6 Done Sent Done false
82252024-02-07T18:58:14.387Z 19 Acked 1018 Read 7 Done Sent Done false
82262024-02-07T18:58:14.387Z 20 Acked 1019 Write 1 Done Sent Done false
82272024-02-07T18:58:14.387Z 21 Acked 1020 Read 4 Done Sent Done false
82282024-02-07T18:58:14.387Z 22 Acked 1021 Flush 0 Done Sent Done false
82292024-02-07T18:58:14.387Z 23 Acked 1022 Flush 0 Done Sent Done false
82302024-02-07T18:58:14.387Z 24 Acked 1023 Write 2 Done Sent Done false
82312024-02-07T18:58:14.387Z 25 Acked 1024 Read 5 Done Sent Done false
82322024-02-07T18:58:14.387Z 26 Acked 1025 Write 3 Done Sent Done false
82332024-02-07T18:58:14.387Z 27 Acked 1026 Write 9 Done Sent Done false
82342024-02-07T18:58:14.387Z 28 Acked 1027 Write 2 Done Sent Done false
82352024-02-07T18:58:14.387Z 29 Acked 1028 Read 7 Done Sent Done false
82362024-02-07T18:58:14.387Z 30 Acked 1029 Write 10 Sent Sent Sent false
82372024-02-07T18:58:14.387Z STATES DS:0 DS:1 DS:2 TOTAL
82382024-02-07T18:58:14.387Z New 0 0 0 0
82392024-02-07T18:58:14.387Z Sent 1 28 1 30
82402024-02-07T18:58:14.387Z Done 29 2 29 60
82412024-02-07T18:58:14.387Z Skipped 0 0 0 0
82422024-02-07T18:58:14.387Z Error 0 0 0 0
82432024-02-07T18:58:14.388ZLast Flush: 1022 0 1022
82442024-02-07T18:58:14.388ZDownstairs last five completed:
82452024-02-07T18:58:14.388ZUpstairs last five completed: 30 29 28 27 26
82462024-02-07T18:58:14.388Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:04.190856983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1431}
82472024-02-07T18:58:14.388ZWrote out file "/tmp/test_repair_verify.data"
82482024-02-07T18:58:14.388Z
82492024-02-07T18:58:14.388ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
82502024-02-07T18:58:14.388ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82512024-02-07T18:58:14.388Z 0 000-019 72 72 72 116 116 116 F F F
82522024-02-07T18:58:14.388Z 1 020-039 74 72 74 120 116 120 F F F <---
82532024-02-07T18:58:14.388Z 2 040-059 74 60 74 118 96 118 T T T <---
82542024-02-07T18:58:14.388Z 3 060-079 71 71 71 113 113 113 T F T <---
82552024-02-07T18:58:14.388Z 4 080-099 74 66 74 118 106 118 F F F <---
82562024-02-07T18:58:14.388Z 5 100-119 72 72 72 116 116 116 F F F
82572024-02-07T18:58:14.388Z 6 120-139 72 72 72 116 116 116 F F F
82582024-02-07T18:58:14.388Z 7 140-159 71 71 71 113 113 113 F F F
82592024-02-07T18:58:14.389Z 8 160-179 74 70 74 120 112 120 F F F <---
82602024-02-07T18:58:14.389Z 9 180-199 70 70 70 112 112 112 T F T <---
82612024-02-07T18:58:14.389Z 10 200-219 74 72 74 120 114 120 F F F <---
82622024-02-07T18:58:14.389Z 11 220-239 72 72 72 116 116 116 F F F
82632024-02-07T18:58:14.389Z 12 240-259 72 72 72 116 116 116 F F F
82642024-02-07T18:58:14.389Z 13 260-279 70 70 70 112 112 112 F F F
82652024-02-07T18:58:14.389Z 14 280-299 67 67 67 107 107 107 F F F
82662024-02-07T18:58:14.389Z 15 300-319 67 67 67 107 107 107 F F F
82672024-02-07T18:58:14.413Z 16 320-339 67 67 67 107 107 107 F F F
82682024-02-07T18:58:14.413Z 17 340-359 74 65 74 120 105 120 F F F <---
82692024-02-07T18:58:14.413Z 18 360-379 72 72 72 116 116 116 F F F
82702024-02-07T18:58:14.413Z 19 380-399 60 60 60 96 96 96 F F F
82712024-02-07T18:58:14.413Z 20 400-419 69 69 69 111 111 111 F F F
82722024-02-07T18:58:14.413Z 21 420-439 72 72 72 115 115 115 F F F
82732024-02-07T18:58:14.413Z 22 440-459 72 72 72 115 115 115 F F F
82742024-02-07T18:58:14.413Z 23 460-479 74 72 74 120 116 120 F F F <---
82752024-02-07T18:58:14.413Z 24 480-499 73 73 73 117 117 117 T F T <---
82762024-02-07T18:58:14.413Z 25 500-519 72 72 72 114 114 114 T F T <---
82772024-02-07T18:58:14.415Z 26 520-539 74 70 74 120 112 120 T F T <---
82782024-02-07T18:58:14.415Z 27 540-559 74 73 74 120 117 120 F F F <---
82792024-02-07T18:58:14.415Z 28 560-579 68 68 68 109 109 109 F F F
82802024-02-07T18:58:14.415Z 29 580-599 60 60 60 96 96 96 F F F
82812024-02-07T18:58:14.415ZMax gen: 74, Max flush: 120
82822024-02-07T18:58:14.415ZError: Difference in extent metadata found!
82832024-02-07T18:58:14.415ZA Difference in extent metadata is expected here
82842024-02-07T18:58:14.415ZOn loop 37
82852024-02-07T18:58:14.415Z
82862024-02-07T18:58:14.415ZVerifying data now
82872024-02-07T18:58:14.415ZNo change after verify
82882024-02-07T18:58:14.415ZLoop: 37 Downstairs dump after verify (and repair):
82892024-02-07T18:58:14.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82902024-02-07T18:58:14.416Z 0 000-019 72 72 72 116 116 116 F F F
82912024-02-07T18:58:14.416Z 1 020-039 74 74 74 120 120 120 F F F
82922024-02-07T18:58:14.416Z 2 040-059 75 75 75 121 121 121 F F F
82932024-02-07T18:58:14.416Z 3 060-079 75 75 75 121 121 121 F F F
82942024-02-07T18:58:14.416Z 4 080-099 74 74 74 118 118 118 F F F
82952024-02-07T18:58:14.416Z 5 100-119 72 72 72 116 116 116 F F F
82962024-02-07T18:58:14.416Z 6 120-139 72 72 72 116 116 116 F F F
82972024-02-07T18:58:14.416Z 7 140-159 71 71 71 113 113 113 F F F
82982024-02-07T18:58:14.416Z 8 160-179 74 74 74 120 120 120 F F F
82992024-02-07T18:58:14.416Z 9 180-199 75 75 75 121 121 121 F F F
83002024-02-07T18:58:14.417Z 10 200-219 74 74 74 120 120 120 F F F
83012024-02-07T18:58:14.417Z 11 220-239 72 72 72 116 116 116 F F F
83022024-02-07T18:58:14.417Z 12 240-259 72 72 72 116 116 116 F F F
83032024-02-07T18:58:14.417Z 13 260-279 70 70 70 112 112 112 F F F
83042024-02-07T18:58:14.417Z 14 280-299 67 67 67 107 107 107 F F F
83052024-02-07T18:58:14.417Z 15 300-319 67 67 67 107 107 107 F F F
83062024-02-07T18:58:14.417Z 16 320-339 67 67 67 107 107 107 F F F
83072024-02-07T18:58:14.417Z 17 340-359 74 74 74 120 120 120 F F F
83082024-02-07T18:58:14.417Z 18 360-379 72 72 72 116 116 116 F F F
83092024-02-07T18:58:14.418Z 19 380-399 60 60 60 96 96 96 F F F
83102024-02-07T18:58:14.418Z 20 400-419 69 69 69 111 111 111 F F F
83112024-02-07T18:58:14.418Z 21 420-439 72 72 72 115 115 115 F F F
83122024-02-07T18:58:14.418Z 22 440-459 72 72 72 115 115 115 F F F
83132024-02-07T18:58:14.418Z 23 460-479 74 74 74 120 120 120 F F F
83142024-02-07T18:58:14.418Z 24 480-499 75 75 75 121 121 121 F F F
83152024-02-07T18:58:14.418Z 25 500-519 75 75 75 121 121 121 F F F
83162024-02-07T18:58:14.418Z 26 520-539 75 75 75 121 121 121 F F F
83172024-02-07T18:58:14.418Z 27 540-559 74 74 74 120 120 120 F F F
83182024-02-07T18:58:14.418Z 28 560-579 68 68 68 109 109 109 F F F
83192024-02-07T18:58:14.418Z 29 580-599 60 60 60 96 96 96 F F F
83202024-02-07T18:58:14.419ZMax gen: 75, Max flush: 121
83212024-02-07T18:58:14.419Z
83222024-02-07T18:58:14.419ZBegin loop 38
83232024-02-07T18:58:14.419ZDownstairs to restart: 1
83242024-02-07T18:58:14.419Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.909988596Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439}
83252024-02-07T18:58:14.419Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
83262024-02-07T18:58:14.419Z,"time":"2024-02-07T18:58:09.910302764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439}
83272024-02-07T18:58:14.419Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910339829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439}
83282024-02-07T18:58:14.419Z{"msg":"Crucible 6814c318-35d0-4b3d-ac14-d3ee7e83d921 has session id: bff6f763-0a9b-45db-9e4d-fe3f2ba20944","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910372694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83292024-02-07T18:58:14.419Z{"msg":"Upstairs opts: Upstairs UUID: 6814c318-35d0-4b3d-ac14-d3ee7e83d921, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.91040588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83302024-02-07T18:58:14.420Z{"msg":"Crucible stats registered with UUID: 6814c318-35d0-4b3d-ac14-d3ee7e83d921","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910432466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83312024-02-07T18:58:14.420Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910446684Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439}
83322024-02-07T18:58:14.420Z{"msg":"6814c318-35d0-4b3d-ac14-d3ee7e83d921 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910467131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83332024-02-07T18:58:14.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910480379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83342024-02-07T18:58:14.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910495588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83352024-02-07T18:58:14.422Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910516415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83362024-02-07T18:58:14.422Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910539682Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83372024-02-07T18:58:14.422Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910601593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83382024-02-07T18:58:14.422Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910631329Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83392024-02-07T18:58:14.422Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910675233Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83402024-02-07T18:58:14.422Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910703869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83412024-02-07T18:58:14.423Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910738485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83422024-02-07T18:58:14.423Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:09.910778309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83432024-02-07T18:58:14.423Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:09.910816764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83442024-02-07T18:58:14.423Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910840301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83452024-02-07T18:58:14.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.910873086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83462024-02-07T18:58:14.424Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:09.911016597Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83472024-02-07T18:58:14.424Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911061711Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83482024-02-07T18:58:14.424Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911112194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83492024-02-07T18:58:14.424Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911151049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83502024-02-07T18:58:14.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911394946Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83512024-02-07T18:58:14.424Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911430551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83522024-02-07T18:58:14.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911446019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83532024-02-07T18:58:14.425Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.911470166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83542024-02-07T18:58:14.425Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912043008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83552024-02-07T18:58:14.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912085573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83562024-02-07T18:58:14.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912584025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83572024-02-07T18:58:14.425Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.91261851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83582024-02-07T18:58:14.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912639058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83592024-02-07T18:58:14.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912677582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83602024-02-07T18:58:14.426Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912701069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83612024-02-07T18:58:14.426Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:09.912721327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83622024-02-07T18:58:14.426Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.912410478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83632024-02-07T18:58:14.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.912561337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83642024-02-07T18:58:14.426Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.913556792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83652024-02-07T18:58:14.426Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.913593607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83662024-02-07T18:58:14.426Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914122136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83672024-02-07T18:58:14.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914676961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83682024-02-07T18:58:14.443Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914717746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83692024-02-07T18:58:14.443Z{"msg":"[0]R flush_numbers[0..12]: [116, 120, 121, 121, 118, 116, 116, 113, 120, 121, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914737723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83702024-02-07T18:58:14.443Z{"msg":"[0]R generation[0..12]: [72, 74, 75, 75, 74, 72, 72, 71, 74, 75, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914771078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83712024-02-07T18:58:14.444Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914790266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83722024-02-07T18:58:14.444Z{"msg":"[1]R flush_numbers[0..12]: [116, 120, 121, 121, 118, 116, 116, 113, 120, 121, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914816852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83732024-02-07T18:58:14.444Z{"msg":"[1]R generation[0..12]: [72, 74, 75, 75, 74, 72, 72, 71, 74, 75, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.91483485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83742024-02-07T18:58:14.444Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914861856Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83752024-02-07T18:58:14.444ZWait for a query_work_queue command to finish before sending IO
83762024-02-07T18:58:14.444Z{"msg":"[2]R flush_numbers[0..12]: [116, 120, 121, 121, 118, 116, 116, 113, 120, 121, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914894372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83772024-02-07T18:58:14.444Z{"msg":"[2]R generation[0..12]: [72, 74, 75, 75, 74, 72, 72, 71, 74, 75, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914920688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83782024-02-07T18:58:14.444Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.914946764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83792024-02-07T18:58:14.444Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.91497862Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83802024-02-07T18:58:14.444Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915012556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83812024-02-07T18:58:14.444Z{"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915030063Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83822024-02-07T18:58:14.445Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915048071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83832024-02-07T18:58:14.445Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915073678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83842024-02-07T18:58:14.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915087245Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83852024-02-07T18:58:14.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915107553Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83862024-02-07T18:58:14.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.91512758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83872024-02-07T18:58:14.445Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915147477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83882024-02-07T18:58:14.445ZError: Difference in extent metadata found!
83892024-02-07T18:58:14.445Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915167485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83902024-02-07T18:58:14.445Z{"msg":"6814c318-35d0-4b3d-ac14-d3ee7e83d921 is now active with session: bff6f763-0a9b-45db-9e4d-fe3f2ba20944","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915181373Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83912024-02-07T18:58:14.445Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915193621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439,"session_id":"bff6f763-0a9b-45db-9e4d-fe3f2ba20944"}
83922024-02-07T18:58:14.446Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:10.915206029Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439}
83932024-02-07T18:58:14.446ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
83942024-02-07T18:58:14.446ZLoading write count information from file "/tmp/test_repair_verify.data"
83952024-02-07T18:58:14.446ZRun Repair workload
83962024-02-07T18:58:14.446Z01/30 Write block 169 len 40960 data: 10 9 9 9 9 7 6 5 5 4
83972024-02-07T18:58:14.446Z02/30 Read block 560 len 40960
83982024-02-07T18:58:14.446Z03/30 Write block 77 len 16384 data: 7 8 6 5
83992024-02-07T18:58:14.447Z04/30 Read block 15 len 40960
84002024-02-07T18:58:14.447Z05/30 Read block 540 len 40960
84012024-02-07T18:58:14.447Z06/30 Read block 534 len 4096
84022024-02-07T18:58:14.447Z07/30 Flush
84032024-02-07T18:58:14.447Z08/30 Write block 365 len 20480 data: 6 8 6 7 6
84042024-02-07T18:58:14.447Z09/30 Write block 480 len 36864 data: 3 3 3 5 4 2 2 4 6
84052024-02-07T18:58:14.447Z10/30 Read block 95 len 20480
84062024-02-07T18:58:14.448Z11/30 Read block 196 len 16384
84072024-02-07T18:58:14.448Z12/30 Read block 23 len 28672
84082024-02-07T18:58:14.448Z13/30 Write block 597 len 4096 data: 3
84092024-02-07T18:58:14.448Z14/30 Read block 507 len 36864
84102024-02-07T18:58:14.448Z15/30 Read block 436 len 24576
84112024-02-07T18:58:14.448Z16/30 Write block 289 len 40960 data: 9 10 8 6 7 6 7 6 8 7
84122024-02-07T18:58:14.449Z17/30 Read block 498 len 16384
84132024-02-07T18:58:14.449Z18/30 Write block 41 len 16384 data: 5 5 5 6
84142024-02-07T18:58:14.450Z19/30 Write block 19 len 20480 data: 10 10 9 10 10
84152024-02-07T18:58:14.450Z20/30 Write block 287 len 36864 data: 10 10 10 11 9 7 8 7 8
84162024-02-07T18:58:14.450Z21/30 Write block 370 len 8192 data: 6 5
84172024-02-07T18:58:14.450Z22/30 Write block 255 len 20480 data: 4 6 5 5 8
84182024-02-07T18:58:14.451Z23/30 Write block 51 len 36864 data: 8 7 8 9 10 10 10 9 8
84192024-02-07T18:58:14.451Z24/30 Read block 546 len 36864
84202024-02-07T18:58:14.451Z25/30 Write block 448 len 16384 data: 6 5 5 6
84212024-02-07T18:58:14.451Z26/30 Read block 427 len 36864
84222024-02-07T18:58:14.452Z27/30 Read block 173 len 36864
84232024-02-07T18:58:14.452Z28/30 Write block 435 len 12288 data: 5 7 9
84242024-02-07T18:58:14.452Z29/30 Read block 396 len 36864
84252024-02-07T18:58:14.452Z30/30 Write block 542 len 4096 data: 12
84262024-02-07T18:58:14.453Z----------------------------------------------------------------
84272024-02-07T18:58:14.453Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
84282024-02-07T18:58:14.454ZStates: Active Active Active
84292024-02-07T18:58:14.454ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84302024-02-07T18:58:14.454Z 1 Acked 1000 Write 10 Done Done Done false
84312024-02-07T18:58:14.455Z 2 Acked 1001 Read 10 Done Done Done false
84322024-02-07T18:58:14.455Z 3 Acked 1002 Write 4 Done Done Done false
84332024-02-07T18:58:14.455Z 4 Acked 1003 Read 10 Done Done Done false
84342024-02-07T18:58:14.455Z 5 Acked 1004 Read 10 Done Sent Done false
84352024-02-07T18:58:14.456Z 6 Acked 1005 Read 1 Done Sent Done false
84362024-02-07T18:58:14.456Z 7 Acked 1006 Flush 0 Done Sent Done false
84372024-02-07T18:58:14.456Z 8 Acked 1007 Write 5 Done Sent Done false
84382024-02-07T18:58:14.456Z 9 Acked 1008 Write 9 Done Sent Done false
84392024-02-07T18:58:14.456Z 10 Acked 1009 Read 5 Done Sent Done false
84402024-02-07T18:58:14.457Z 11 Acked 1010 Read 4 Done Sent Done false
84412024-02-07T18:58:14.457Z 12 Acked 1011 Read 7 Done Sent Done false
84422024-02-07T18:58:14.457Z 13 Acked 1012 Write 1 Done Sent Done false
84432024-02-07T18:58:14.457Z 14 Acked 1013 Read 9 Done Sent Done false
84442024-02-07T18:58:14.457Z 15 Acked 1014 Read 6 Done Sent Done false
84452024-02-07T18:58:14.458Z 16 Acked 1015 Write 10 Done Sent Done false
84462024-02-07T18:58:14.458Z 17 Acked 1016 Read 4 Done Sent Done false
84472024-02-07T18:58:14.458Z 18 Acked 1017 Write 4 Done Sent Done false
84482024-02-07T18:58:14.458Z 19 Acked 1018 Write 5 Done Sent Done false
84492024-02-07T18:58:14.458Z 20 Acked 1019 Write 9 Done Sent Done false
84502024-02-07T18:58:14.458Z 21 Acked 1020 Write 2 Done Sent Done false
84512024-02-07T18:58:14.459Z 22 Acked 1021 Write 5 Done Sent Done false
84522024-02-07T18:58:14.459Z 23 Acked 1022 Write 9 Done Sent Done false
84532024-02-07T18:58:14.459Z 24 Acked 1023 Read 9 Done Sent Done false
84542024-02-07T18:58:14.459Z 25 Acked 1024 Write 4 Done Sent Done false
84552024-02-07T18:58:14.460Z 26 Acked 1025 Read 9 Done Sent Done false
84562024-02-07T18:58:14.460Z 27 Acked 1026 Read 9 Done Sent Done false
84572024-02-07T18:58:14.460Z 28 Acked 1027 Write 3 Done Sent Done false
84582024-02-07T18:58:14.460Z 29 Acked 1028 Read 9 Done Sent Done false
84592024-02-07T18:58:14.460Z 30 Acked 1029 Write 1 Sent Sent Sent false
84602024-02-07T18:58:14.461Z STATES DS:0 DS:1 DS:2 TOTAL
84612024-02-07T18:58:14.461Z New 0 0 0 0
84622024-02-07T18:58:14.461Z Sent 1 26 1 28
84632024-02-07T18:58:14.461Z Done 29 4 29 62
84642024-02-07T18:58:14.461Z Skipped 0 0 0 0
84652024-02-07T18:58:14.462Z Error 0 0 0 0
84662024-02-07T18:58:14.462ZLast Flush: 1006 0 1006
84672024-02-07T18:58:14.489ZDownstairs last five completed:
84682024-02-07T18:58:14.489ZUpstairs last five completed: 30 29 28 27 26
84692024-02-07T18:58:14.489Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:10.945423982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1439}
84702024-02-07T18:58:14.489ZWrote out file "/tmp/test_repair_verify.data"
84712024-02-07T18:58:14.489Z
84722024-02-07T18:58:14.489ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
84732024-02-07T18:58:14.490ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84742024-02-07T18:58:14.490Z 0 000-019 72 72 72 116 116 116 T F T <---
84752024-02-07T18:58:14.490Z 1 020-039 74 74 74 120 120 120 T F T <---
84762024-02-07T18:58:14.490Z 2 040-059 75 75 75 121 121 121 T F T <---
84772024-02-07T18:58:14.490Z 3 060-079 76 75 76 122 121 122 F T F <---
84782024-02-07T18:58:14.490Z 4 080-099 76 74 76 122 118 122 F T F <---
84792024-02-07T18:58:14.490Z 5 100-119 72 72 72 116 116 116 F F F
84802024-02-07T18:58:14.491Z 6 120-139 72 72 72 116 116 116 F F F
84812024-02-07T18:58:14.491Z 7 140-159 71 71 71 113 113 113 F F F
84822024-02-07T18:58:14.491Z 8 160-179 76 74 76 122 120 122 F T F <---
84832024-02-07T18:58:14.491Z 9 180-199 75 75 75 121 121 121 F F F
84842024-02-07T18:58:14.491Z 10 200-219 74 74 74 120 120 120 F F F
84852024-02-07T18:58:14.491Z 11 220-239 72 72 72 116 116 116 F F F
84862024-02-07T18:58:14.492Z 12 240-259 72 72 72 116 116 116 T F T <---
84872024-02-07T18:58:14.492Z 13 260-279 70 70 70 112 112 112 F F F
84882024-02-07T18:58:14.492Z 14 280-299 67 67 67 107 107 107 T F T <---
84892024-02-07T18:58:14.492Z 15 300-319 67 67 67 107 107 107 F F F
84902024-02-07T18:58:14.492Z 16 320-339 67 67 67 107 107 107 F F F
84912024-02-07T18:58:14.492Z 17 340-359 74 74 74 120 120 120 F F F
84922024-02-07T18:58:14.492Z 18 360-379 72 72 72 116 116 116 T F T <---
84932024-02-07T18:58:14.492Z 19 380-399 60 60 60 96 96 96 F F F
84942024-02-07T18:58:14.492Z 20 400-419 69 69 69 111 111 111 F F F
84952024-02-07T18:58:14.493Z 21 420-439 72 72 72 115 115 115 T F T <---
84962024-02-07T18:58:14.493Z 22 440-459 72 72 72 115 115 115 T F T <---
84972024-02-07T18:58:14.493Z 23 460-479 74 74 74 120 120 120 F F F
84982024-02-07T18:58:14.493Z 24 480-499 75 75 75 121 121 121 T F T <---
84992024-02-07T18:58:14.493Z 25 500-519 75 75 75 121 121 121 F F F
85002024-02-07T18:58:14.493Z 26 520-539 75 75 75 121 121 121 F F F
85012024-02-07T18:58:14.494Z 27 540-559 74 74 74 120 120 120 T F T <---
85022024-02-07T18:58:14.494Z 28 560-579 68 68 68 109 109 109 F F F
85032024-02-07T18:58:14.494Z 29 580-599 60 60 60 96 96 96 T F T <---
85042024-02-07T18:58:14.494ZMax gen: 76, Max flush: 122
85052024-02-07T18:58:14.494ZA Difference in extent metadata is expected here
85062024-02-07T18:58:14.494ZOn loop 38
85072024-02-07T18:58:14.495Z
85082024-02-07T18:58:14.495ZVerifying data now
85092024-02-07T18:58:19.659ZNo change after verify
85102024-02-07T18:58:19.659ZLoop: 38 Downstairs dump after verify (and repair):
85112024-02-07T18:58:19.680ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85122024-02-07T18:58:19.680Z 0 000-019 77 77 77 123 123 123 F F F
85132024-02-07T18:58:19.680Z 1 020-039 77 77 77 123 123 123 F F F
85142024-02-07T18:58:19.680Z 2 040-059 77 77 77 123 123 123 F F F
85152024-02-07T18:58:19.681Z 3 060-079 76 76 76 122 122 122 F F F
85162024-02-07T18:58:19.681Z 4 080-099 76 76 76 122 122 122 F F F
85172024-02-07T18:58:19.681Z 5 100-119 72 72 72 116 116 116 F F F
85182024-02-07T18:58:19.681Z 6 120-139 72 72 72 116 116 116 F F F
85192024-02-07T18:58:19.681Z 7 140-159 71 71 71 113 113 113 F F F
85202024-02-07T18:58:19.682Z 8 160-179 76 76 76 122 122 122 F F F
85212024-02-07T18:58:19.682Z 9 180-199 75 75 75 121 121 121 F F F
85222024-02-07T18:58:19.682Z 10 200-219 74 74 74 120 120 120 F F F
85232024-02-07T18:58:19.682Z 11 220-239 72 72 72 116 116 116 F F F
85242024-02-07T18:58:19.682Z 12 240-259 77 77 77 123 123 123 F F F
85252024-02-07T18:58:19.682Z 13 260-279 70 70 70 112 112 112 F F F
85262024-02-07T18:58:19.683Z 14 280-299 77 77 77 123 123 123 F F F
85272024-02-07T18:58:19.683Z 15 300-319 67 67 67 107 107 107 F F F
85282024-02-07T18:58:19.683Z 16 320-339 67 67 67 107 107 107 F F F
85292024-02-07T18:58:19.683Z 17 340-359 74 74 74 120 120 120 F F F
85302024-02-07T18:58:19.683Z 18 360-379 77 77 77 123 123 123 F F F
85312024-02-07T18:58:19.683Z 19 380-399 60 60 60 96 96 96 F F F
85322024-02-07T18:58:19.683Z 20 400-419 69 69 69 111 111 111 F F F
85332024-02-07T18:58:19.683Z 21 420-439 77 77 77 123 123 123 F F F
85342024-02-07T18:58:19.683Z 22 440-459 77 77 77 123 123 123 F F F
85352024-02-07T18:58:19.683Z 23 460-479 74 74 74 120 120 120 F F F
85362024-02-07T18:58:19.684Z 24 480-499 77 77 77 123 123 123 F F F
85372024-02-07T18:58:19.684Z 25 500-519 75 75 75 121 121 121 F F F
85382024-02-07T18:58:19.684Z 26 520-539 75 75 75 121 121 121 F F F
85392024-02-07T18:58:19.684Z 27 540-559 77 77 77 123 123 123 F F F
85402024-02-07T18:58:19.684Z 28 560-579 68 68 68 109 109 109 F F F
85412024-02-07T18:58:19.684Z 29 580-599 77 77 77 123 123 123 F F F
85422024-02-07T18:58:19.684ZMax gen: 77, Max flush: 123
85432024-02-07T18:58:19.684Z
85442024-02-07T18:58:19.684ZBegin loop 39
85452024-02-07T18:58:19.684ZDownstairs to restart: 0
85462024-02-07T18:58:19.684Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.708970099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447}
85472024-02-07T18:58:19.684Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709231084Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447}
85482024-02-07T18:58:19.684Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709271238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447}
85492024-02-07T18:58:19.684ZCrucible runtime is spawned
85502024-02-07T18:58:19.685Z{"msg":"Crucible b279fcf8-aca5-4d87-90e5-617a48410111 has session id: b9ea61e0-b271-41db-95d5-8b6662f20d84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709322162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85512024-02-07T18:58:19.685Z{"msg":"Upstairs opts: Upstairs UUID: b279fcf8-aca5-4d87-90e5-617a48410111, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709370525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85522024-02-07T18:58:19.685Z{"msg":"Crucible stats registered with UUID: b279fcf8-aca5-4d87-90e5-617a48410111","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709396812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85532024-02-07T18:58:19.685Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.70941035Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447}
85542024-02-07T18:58:19.685Z{"msg":"b279fcf8-aca5-4d87-90e5-617a48410111 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709492839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85552024-02-07T18:58:19.685Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709525274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85562024-02-07T18:58:19.685Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709561839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85572024-02-07T18:58:19.685Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709600294Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85582024-02-07T18:58:19.685Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709640548Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85592024-02-07T18:58:19.685Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709668615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85602024-02-07T18:58:19.685Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.7097037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85612024-02-07T18:58:19.685Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709737626Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85622024-02-07T18:58:19.685Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.70977641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85632024-02-07T18:58:19.686Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709807676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85642024-02-07T18:58:19.686Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:16.709836802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85652024-02-07T18:58:19.686Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:16.709866978Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85662024-02-07T18:58:19.686Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709907583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85672024-02-07T18:58:19.686Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:16.709953956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85682024-02-07T18:58:19.686Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.709988052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85692024-02-07T18:58:19.686Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710032135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85702024-02-07T18:58:19.686Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.71007398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85712024-02-07T18:58:19.686Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710107675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85722024-02-07T18:58:19.686Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710490024Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85732024-02-07T18:58:19.686Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710531938Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85742024-02-07T18:58:19.686Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710724592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85752024-02-07T18:58:19.686Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710766606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85762024-02-07T18:58:19.686Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.710981787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85772024-02-07T18:58:19.687Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711093192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85782024-02-07T18:58:19.687Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711466921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85792024-02-07T18:58:19.687Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711501737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85802024-02-07T18:58:19.687Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711518385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85812024-02-07T18:58:19.687Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711590105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85822024-02-07T18:58:19.687Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711634249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85832024-02-07T18:58:19.687Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:16.711661295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85842024-02-07T18:58:19.687Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.711451053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85852024-02-07T18:58:19.687Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.711584995Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85862024-02-07T18:58:19.687Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.712378448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85872024-02-07T18:58:19.687Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.712417842Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85882024-02-07T18:58:19.687Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.71295212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85892024-02-07T18:58:19.687Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.71354534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85902024-02-07T18:58:19.687Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713593963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85912024-02-07T18:58:19.687Z{"msg":"[0]R flush_numbers[0..12]: [123, 123, 123, 122, 122, 116, 116, 113, 122, 121, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.71362385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85922024-02-07T18:58:19.687Z{"msg":"[0]R generation[0..12]: [77, 77, 77, 76, 76, 72, 72, 71, 76, 75, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713650406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85932024-02-07T18:58:19.687Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713678172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85942024-02-07T18:58:19.687Z{"msg":"[1]R flush_numbers[0..12]: [123, 123, 123, 122, 122, 116, 116, 113, 122, 121, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713704708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85952024-02-07T18:58:19.687Z{"msg":"[1]R generation[0..12]: [77, 77, 77, 76, 76, 72, 72, 71, 76, 75, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713730665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85962024-02-07T18:58:19.688Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]"Wait for a query_work_queue command to finish before sending IO
85972024-02-07T18:58:19.688Z,"v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713759971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85982024-02-07T18:58:19.688Z{"msg":"[2]R flush_numbers[0..12]: [123, 123, 123, 122, 122, 116, 116, 113, 122, 121, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713787047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
85992024-02-07T18:58:19.688Z{"msg":"[2]R generation[0..12]: [77, 77, 77, 76, 76, 72, 72, 71, 76, 75, 74, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713812284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86002024-02-07T18:58:19.688Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.71383893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86012024-02-07T18:58:19.688Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713863677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86022024-02-07T18:58:19.688Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713881025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86032024-02-07T18:58:19.688Z{"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713905491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86042024-02-07T18:58:19.688Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713922339Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86052024-02-07T18:58:19.688Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713947716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86062024-02-07T18:58:19.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.713963893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86072024-02-07T18:58:19.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.71398869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86082024-02-07T18:58:19.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.714007807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86092024-02-07T18:58:19.688Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.714026125Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86102024-02-07T18:58:19.689Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.714050802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86112024-02-07T18:58:20.112Z{"msg":"b279fcf8-aca5-4d87-90e5-617a48410111 is now active with session: b9ea61e0-b271-41db-95d5-8b6662f20d84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.714076618Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86122024-02-07T18:58:20.112Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.714099895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447,"session_id":"b9ea61e0-b271-41db-95d5-8b6662f20d84"}
86132024-02-07T18:58:20.112Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:17.714123172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447}
86142024-02-07T18:58:20.112ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86152024-02-07T18:58:20.112ZLoading write count information from file "/tmp/test_repair_verify.data"
86162024-02-07T18:58:20.112ZRun Repair workload
86172024-02-07T18:58:20.112Z01/30 Write block 95 len 20480 data: 9 9 8 7 6
86182024-02-07T18:58:20.112Z02/30 Read block 556 len 8192
86192024-02-07T18:58:20.112Z03/30 Write block 221 len 20480 data: 4 4 5 6 4
86202024-02-07T18:58:20.112Z04/30 Read block 65 len 16384
86212024-02-07T18:58:20.112Z05/30 Write block 532 len 28672 data: 6 7 9 10 11 12 10
86222024-02-07T18:58:20.113Z06/30 Read block 115 len 24576
86232024-02-07T18:58:20.113Z07/30 Write block 313 len 40960 data: 6 7 10 12 10 10 8 7 6 7
86242024-02-07T18:58:20.113Z08/30 Read block 470 len 32768
86252024-02-07T18:58:20.113Z09/30 Write block 178 len 4096 data: 5
86262024-02-07T18:58:20.113Z10/30 Read block 127 len 32768
86272024-02-07T18:58:20.113Z11/30 Write block 277 len 24576 data: 2 3 3 3 3 4
86282024-02-07T18:58:20.113Z12/30 Flush
86292024-02-07T18:58:20.113Z13/30 Write block 240 len 12288 data: 13 14 14
86302024-02-07T18:58:20.113Z14/30 Read block 196 len 12288
86312024-02-07T18:58:20.113Z15/30 Flush
86322024-02-07T18:58:20.113Z16/30 Write block 402 len 4096 data: 5
86332024-02-07T18:58:20.113Z17/30 Write block 116 len 32768 data: 6 8 9 10 9 9 11 9
86342024-02-07T18:58:20.113Z18/30 Read block 366 len 4096
86352024-02-07T18:58:20.113Z19/30 Read block 394 len 40960
86362024-02-07T18:58:20.113Z20/30 Write block 280 len 24576 data: 4 4 5 4 6 7
86372024-02-07T18:58:20.113Z21/30 Write block 309 len 32768 data: 6 7 7 5 7 8 11 13
86382024-02-07T18:58:20.113Z22/30 Flush
86392024-02-07T18:58:20.113Z23/30 Write block 357 len 32768 data: 2 4 6 8 9 10 9 7
86402024-02-07T18:58:20.114Z24/30 Write block 564 len 20480 data: 5 5 5 5 5
86412024-02-07T18:58:20.114Z25/30 Flush
86422024-02-07T18:58:20.114Z26/30 Write block 567 len 36864 data: 6 6 6 7 7 5 5 4 4
86432024-02-07T18:58:20.114Z27/30 Read block 24 len 28672
86442024-02-07T18:58:20.114Z28/30 Write block 128 len 32768 data: 8 8 7 6 6 6 6 4
86452024-02-07T18:58:20.114Z29/30 Write block 412 len 4096 data: 7
86462024-02-07T18:58:20.114Z30/30 Read block 305 len 32768
86472024-02-07T18:58:20.114ZError: Difference in extent metadata found!
86482024-02-07T18:58:20.114Z----------------------------------------------------------------
86492024-02-07T18:58:20.114Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
86502024-02-07T18:58:20.114ZStates: Active Active Active
86512024-02-07T18:58:20.114ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86522024-02-07T18:58:20.114Z 1 Acked 1000 Write 5 Done Done Done false
86532024-02-07T18:58:20.114Z 2 Acked 1001 Read 2 Done Done Done false
86542024-02-07T18:58:20.114Z 3 Acked 1002 Write 5 Done Done Done false
86552024-02-07T18:58:20.114Z 4 Acked 1003 Read 4 Sent Done Done false
86562024-02-07T18:58:20.114Z 5 Acked 1004 Write 7 Sent Done Done false
86572024-02-07T18:58:20.114Z 6 Acked 1005 Read 6 Sent Done Done false
86582024-02-07T18:58:20.114Z 7 Acked 1006 Write 10 Sent Done Done false
86592024-02-07T18:58:20.115Z 8 Acked 1007 Read 8 Sent Done Done false
86602024-02-07T18:58:20.115Z 9 Acked 1008 Write 1 Sent Done Done false
86612024-02-07T18:58:20.115Z 10 Acked 1009 Read 8 Sent Done Done false
86622024-02-07T18:58:20.115Z 11 Acked 1010 Write 6 Sent Done Done false
86632024-02-07T18:58:20.115Z 12 Acked 1011 Flush 0 Sent Done Done false
86642024-02-07T18:58:20.115Z 13 Acked 1012 Write 3 Sent Done Done false
86652024-02-07T18:58:20.115Z 14 Acked 1013 Read 3 Sent Done Done false
86662024-02-07T18:58:20.115Z 15 Acked 1014 Flush 0 Sent Done Done false
86672024-02-07T18:58:20.115Z 16 Acked 1015 Write 1 Sent Done Done false
86682024-02-07T18:58:20.115Z 17 Acked 1016 Write 8 Sent Done Done false
86692024-02-07T18:58:20.115Z 18 Acked 1017 Read 1 Sent Done Done false
86702024-02-07T18:58:20.115Z 19 Acked 1018 Read 10 Sent Done Done false
86712024-02-07T18:58:20.115Z 20 Acked 1019 Write 6 Sent Done Done false
86722024-02-07T18:58:20.115Z 21 Acked 1020 Write 8 Sent Done Done false
86732024-02-07T18:58:20.115Z 22 Acked 1021 Flush 0 Sent Done Done false
86742024-02-07T18:58:20.115Z 23 Acked 1022 Write 8 Sent Done Done false
86752024-02-07T18:58:20.115Z 24 Acked 1023 Write 5 Sent Done Done false
86762024-02-07T18:58:20.115Z 25 Acked 1024 Flush 0 Sent Done Done false
86772024-02-07T18:58:20.115Z 26 Acked 1025 Write 9 Sent Done Done false
86782024-02-07T18:58:20.115Z 27 Acked 1026 Read 7 Sent Done Done false
86792024-02-07T18:58:20.115Z 28 Acked 1027 Write 8 Sent Done Done false
86802024-02-07T18:58:20.116Z 29 Acked 1028 Write 1 Sent Done Done false
86812024-02-07T18:58:20.116Z 30 Acked 1029 Read 8 Sent Sent Done false
86822024-02-07T18:58:20.116Z STATES DS:0 DS:1 DS:2 TOTAL
86832024-02-07T18:58:20.116Z New 0 0 0 0
86842024-02-07T18:58:20.116Z Sent 27 1 0 28
86852024-02-07T18:58:20.116Z Done 3 29 30 62
86862024-02-07T18:58:20.116Z Skipped 0 0 0 0
86872024-02-07T18:58:20.116Z Error 0 0 0 0
86882024-02-07T18:58:20.116ZLast Flush: 0 1024 1024
86892024-02-07T18:58:20.116ZDownstairs last five completed:
86902024-02-07T18:58:20.116ZUpstairs last five completed: 30 29 28 27 26
86912024-02-07T18:58:20.116Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:17.749020882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1447}
86922024-02-07T18:58:20.116ZWrote out file "/tmp/test_repair_verify.data"
86932024-02-07T18:58:20.116Z
86942024-02-07T18:58:20.116ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
86952024-02-07T18:58:20.116ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86962024-02-07T18:58:20.116Z 0 000-019 77 77 77 123 123 123 F F F
86972024-02-07T18:58:20.116Z 1 020-039 77 77 77 123 123 123 F F F
86982024-02-07T18:58:20.116Z 2 040-059 77 77 77 123 123 123 F F F
86992024-02-07T18:58:20.116Z 3 060-079 76 76 76 122 122 122 F F F
87002024-02-07T18:58:20.116Z 4 080-099 76 78 78 122 124 124 T F F <---
87012024-02-07T18:58:20.116Z 5 100-119 72 78 78 116 126 126 F F F <---
87022024-02-07T18:58:20.116Z 6 120-139 72 78 78 116 126 126 F T T <---
87032024-02-07T18:58:20.116Z 7 140-159 71 71 71 113 113 113 F F F
87042024-02-07T18:58:20.117Z 8 160-179 76 78 78 122 124 124 F F F <---
87052024-02-07T18:58:20.117Z 9 180-199 75 75 75 121 121 121 F F F
87062024-02-07T18:58:20.117Z 10 200-219 74 74 74 120 120 120 F F F
87072024-02-07T18:58:20.117Z 11 220-239 72 78 78 116 124 124 T F F <---
87082024-02-07T18:58:20.117Z 12 240-259 77 78 78 123 125 125 F F F <---
87092024-02-07T18:58:20.117Z 13 260-279 70 78 78 112 124 124 F F F <---
87102024-02-07T18:58:20.117Z 14 280-299 77 78 78 123 126 126 F F F <---
87112024-02-07T18:58:20.129Z 15 300-319 67 78 78 107 126 126 F F F <---
87122024-02-07T18:58:20.129Z 16 320-339 67 78 78 107 124 124 F F F <---
87132024-02-07T18:58:20.129Z 17 340-359 74 78 78 120 127 127 F F F <---
87142024-02-07T18:58:20.129Z 18 360-379 77 78 78 123 127 127 F F F <---
87152024-02-07T18:58:20.129Z 19 380-399 60 60 60 96 96 96 F F F
87162024-02-07T18:58:20.129Z 20 400-419 69 78 78 111 126 126 F T T <---
87172024-02-07T18:58:20.129Z 21 420-439 77 77 77 123 123 123 F F F
87182024-02-07T18:58:20.129Z 22 440-459 77 77 77 123 123 123 F F F
87192024-02-07T18:58:20.129Z 23 460-479 74 74 74 120 120 120 F F F
87202024-02-07T18:58:20.129Z 24 480-499 77 77 77 123 123 123 F F F
87212024-02-07T18:58:20.129Z 25 500-519 75 75 75 121 121 121 F F F
87222024-02-07T18:58:20.129Z 26 520-539 75 78 78 121 124 124 F F F <---
87232024-02-07T18:58:20.129Z 27 540-559 77 77 77 123 123 123 F F F
87242024-02-07T18:58:20.129Z 28 560-579 68 78 78 109 127 127 F T T <---
87252024-02-07T18:58:20.129Z 29 580-599 77 77 77 123 123 123 F F F
87262024-02-07T18:58:20.129ZMax gen: 78, Max flush: 127
87272024-02-07T18:58:20.129ZA Difference in extent metadata is expected here
87282024-02-07T18:58:20.129ZOn loop 39
87292024-02-07T18:58:20.129Z
87302024-02-07T18:58:20.129ZVerifying data now
87312024-02-07T18:58:24.252ZNo change after verify
87322024-02-07T18:58:24.252ZLoop: 39 Downstairs dump after verify (and repair):
87332024-02-07T18:58:24.271ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87342024-02-07T18:58:24.271Z 0 000-019 77 77 77 123 123 123 F F F
87352024-02-07T18:58:24.271Z 1 020-039 77 77 77 123 123 123 F F F
87362024-02-07T18:58:24.271Z 2 040-059 77 77 77 123 123 123 F F F
87372024-02-07T18:58:24.271Z 3 060-079 76 76 76 122 122 122 F F F
87382024-02-07T18:58:24.271Z 4 080-099 78 78 78 124 124 124 F F F
87392024-02-07T18:58:24.271Z 5 100-119 78 78 78 126 126 126 F F F
87402024-02-07T18:58:24.271Z 6 120-139 79 79 79 128 128 128 F F F
87412024-02-07T18:58:24.274Z 7 140-159 71 71 71 113 113 113 F F F
87422024-02-07T18:58:24.274Z 8 160-179 78 78 78 124 124 124 F F F
87432024-02-07T18:58:24.274Z 9 180-199 75 75 75 121 121 121 F F F
87442024-02-07T18:58:24.274Z 10 200-219 74 74 74 120 120 120 F F F
87452024-02-07T18:58:24.274Z 11 220-239 78 78 78 124 124 124 F F F
87462024-02-07T18:58:24.274Z 12 240-259 78 78 78 125 125 125 F F F
87472024-02-07T18:58:24.274Z 13 260-279 78 78 78 124 124 124 F F F
87482024-02-07T18:58:24.274Z 14 280-299 78 78 78 126 126 126 F F F
87492024-02-07T18:58:24.274Z 15 300-319 78 78 78 126 126 126 F F F
87502024-02-07T18:58:24.275Z 16 320-339 78 78 78 124 124 124 F F F
87512024-02-07T18:58:24.275Z 17 340-359 78 78 78 127 127 127 F F F
87522024-02-07T18:58:24.275Z 18 360-379 78 78 78 127 127 127 F F F
87532024-02-07T18:58:24.275Z 19 380-399 60 60 60 96 96 96 F F F
87542024-02-07T18:58:24.275Z 20 400-419 79 79 79 128 128 128 F F F
87552024-02-07T18:58:24.275Z 21 420-439 77 77 77 123 123 123 F F F
87562024-02-07T18:58:24.275Z 22 440-459 77 77 77 123 123 123 F F F
87572024-02-07T18:58:24.275Z 23 460-479 74 74 74 120 120 120 F F F
87582024-02-07T18:58:24.275Z 24 480-499 77 77 77 123 123 123 F F F
87592024-02-07T18:58:24.275Z 25 500-519 75 75 75 121 121 121 F F F
87602024-02-07T18:58:24.275Z 26 520-539 78 78 78 124 124 124 F F F
87612024-02-07T18:58:24.275Z 27 540-559 77 77 77 123 123 123 F F F
87622024-02-07T18:58:24.275Z 28 560-579 79 79 79 128 128 128 F F F
87632024-02-07T18:58:24.275Z 29 580-599 77 77 77 123 123 123 F F F
87642024-02-07T18:58:24.275ZMax gen: 79, Max flush: 128
87652024-02-07T18:58:24.275Z
87662024-02-07T18:58:24.275ZBegin loop 40
87672024-02-07T18:58:24.275ZDownstairs to restart: 0
87682024-02-07T18:58:24.289Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530057404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455}
87692024-02-07T18:58:24.289Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530369201Z","hostname":"Crucible runtime is spawned
87702024-02-07T18:58:24.289Zip-10-150-1-187.us-west-2.compute.internal","pid":1455}
87712024-02-07T18:58:24.295Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530420845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455}
87722024-02-07T18:58:24.295Z{"msg":"Crucible 4388035e-6aeb-4163-bf96-6aa438a53806 has session id: 6482d533-fc24-4fad-83b8-9026a63ab50f","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530450821Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87732024-02-07T18:58:24.295Z{"msg":"Upstairs opts: Upstairs UUID: 4388035e-6aeb-4163-bf96-6aa438a53806, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530507043Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87742024-02-07T18:58:24.295Z{"msg":"Crucible stats registered with UUID: 4388035e-6aeb-4163-bf96-6aa438a53806","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530534009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87752024-02-07T18:58:24.295Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530556126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455}
87762024-02-07T18:58:24.295Z{"msg":"4388035e-6aeb-4163-bf96-6aa438a53806 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530577953Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87772024-02-07T18:58:24.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.53060208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87782024-02-07T18:58:24.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530617348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87792024-02-07T18:58:24.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530631606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87802024-02-07T18:58:24.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530646044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87812024-02-07T18:58:24.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530667641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87822024-02-07T18:58:24.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530685099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87832024-02-07T18:58:24.295Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530720974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87842024-02-07T18:58:24.295Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530738971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87852024-02-07T18:58:24.295Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530760779Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87862024-02-07T18:58:24.295Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:23.530941914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87872024-02-07T18:58:24.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.530979849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87882024-02-07T18:58:24.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531005925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87892024-02-07T18:58:24.295Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:23.531055129Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87902024-02-07T18:58:24.295Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:23.531094463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87912024-02-07T18:58:24.295Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.53111928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87922024-02-07T18:58:24.296Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531150676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87932024-02-07T18:58:24.296Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531184341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87942024-02-07T18:58:24.296Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531615013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87952024-02-07T18:58:24.296Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531668716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87962024-02-07T18:58:24.296Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531703111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87972024-02-07T18:58:24.296Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.531727168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87982024-02-07T18:58:24.296Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532210143Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
87992024-02-07T18:58:24.296Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532262625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88002024-02-07T18:58:24.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532884211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88012024-02-07T18:58:24.296Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532919297Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88022024-02-07T18:58:24.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532934754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88032024-02-07T18:58:24.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532948023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88042024-02-07T18:58:24.296Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.53296957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88052024-02-07T18:58:24.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:23.532984078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88062024-02-07T18:58:25.297Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.532564174Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88072024-02-07T18:58:25.297Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.532804341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88082024-02-07T18:58:25.334Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.533795177Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88092024-02-07T18:58:25.334Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.533831173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88102024-02-07T18:58:25.334Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.534302989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88112024-02-07T18:58:25.334Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.534862433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88122024-02-07T18:58:25.334Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.534896898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88132024-02-07T18:58:25.334Z{"msg":"[0]R flush_numbers[0..12]: [123, 123, 123, 122, 124, 126, 128, 113, 124, 121, 120, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.534915416Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88142024-02-07T18:58:25.334Z{"msg":"[0]R generation[0..12]: [77, 77, 77, 76, 78, 78, 79, 71, 78, 75, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.534946911Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88152024-02-07T18:58:25.334Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.534977597Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88162024-02-07T18:58:25.334Z{"msg":"[1]R flush_numbers[0..12]: [123, 123, 123, 122, 124, 126, 128, 113, 124, 121, 120, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535019782Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88172024-02-07T18:58:25.334Z{"msg":"[1]R generation[0..12]: [77, 77, 77, 76, 78, 78, 79, 71, 78, 75, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535043298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88182024-02-07T18:58:25.334Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535061586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88192024-02-07T18:58:25.335Z{"msg":"[2]R flush_numbers[0..12]: [123, 123, 123, 122, 124, 126, 128, 113, 124, 121, 120, 124]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
88202024-02-07T18:58:25.335Z,"time":"2024-02-07T18:58:24.535089892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88212024-02-07T18:58:25.335Z{"msg":"[2]R generation[0..12]: [77, 77, 77, 76, 78, 78, 79, 71, 78, 75, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.53510873Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88222024-02-07T18:58:25.335Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535127337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88232024-02-07T18:58:25.335Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535144255Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88242024-02-07T18:58:25.335Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535169641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88252024-02-07T18:58:25.335Z{"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535187239Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88262024-02-07T18:58:25.335Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535212376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88272024-02-07T18:58:25.335Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535229623Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88282024-02-07T18:58:25.335Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535246411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88292024-02-07T18:58:25.335Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535271398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88302024-02-07T18:58:25.335Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535288665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88312024-02-07T18:58:25.335Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535320701Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88322024-02-07T18:58:25.335Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535338159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88332024-02-07T18:58:25.335Z{"msg":"4388035e-6aeb-4163-bf96-6aa438a53806 is now active with session: 6482d533-fc24-4fad-83b8-9026a63ab50f","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535363455Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88342024-02-07T18:58:25.335Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.535378993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455,"session_id":"6482d533-fc24-4fad-83b8-9026a63ab50f"}
88352024-02-07T18:58:25.336Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:24.53540288Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455}
88362024-02-07T18:58:25.336ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88372024-02-07T18:58:25.336ZLoading write count information from file "/tmp/test_repair_verify.data"
88382024-02-07T18:58:25.336ZRun Repair workload
88392024-02-07T18:58:25.336Z01/30 Write block 301 len 8192 data: 4 5
88402024-02-07T18:58:25.336Z02/30 Read block 524 len 12288
88412024-02-07T18:58:25.336Z03/30 Read block 441 len 24576
88422024-02-07T18:58:25.336Z04/30 Write block 501 len 36864 data: 4 7 9 9 10 7 6 7 7
88432024-02-07T18:58:25.336Z05/30 Read block 408 len 36864
88442024-02-07T18:58:25.336Z06/30 Flush
88452024-02-07T18:58:25.336Z07/30 Write block 514 len 32768 data: 8 7 6 4 6 6 7 5
88462024-02-07T18:58:25.336Z08/30 Flush
88472024-02-07T18:58:25.348Z09/30 Write block 271 len 40960 data: 4 4 4 3 3 3 3 4 4 5
88482024-02-07T18:58:25.348Z10/30 Read block 315 len 32768
88492024-02-07T18:58:25.348Z11/30 Write block 220 len 4096 data: 4
88502024-02-07T18:58:25.348Z12/30 Read block 200 len 8192
88512024-02-07T18:58:25.348Z13/30 Read block 337 len 24576
88522024-02-07T18:58:25.348Z14/30 Write block 211 len 8192 data: 6 6
88532024-02-07T18:58:25.348Z15/30 Write block 67 len 8192 data: 9 9
88542024-02-07T18:58:25.348Z16/30 Write block 19 len 12288 data: 11 11 10
88552024-02-07T18:58:25.348Z17/30 Read block 250 len 24576
88562024-02-07T18:58:25.348Z18/30 Read block 556 len 40960
88572024-02-07T18:58:25.348Z19/30 Flush
88582024-02-07T18:58:25.348Z20/30 Write block 560 len 28672 data: 6 4 3 3 6 6 6
88592024-02-07T18:58:25.349Z21/30 Read block 18 len 4096
88602024-02-07T18:58:25.349Z22/30 Flush
88612024-02-07T18:58:25.349Z23/30 Write block 407 len 36864 data: 5 5 7 10 11 8 10 8 8
88622024-02-07T18:58:25.349Z24/30 Read block 442 len 28672
88632024-02-07T18:58:25.349Z25/30 Flush
88642024-02-07T18:58:25.349Z26/30 Write block 147 len 32768 data: 11 11 13 8 8 8 9 8
88652024-02-07T18:58:25.349Z27/30 Write block 4 len 40960 data: 5 7 6 9 8 9 7 7 8 8
88662024-02-07T18:58:25.349Z28/30 Read block 424 len 20480
88672024-02-07T18:58:25.349Z29/30 Write block 328 len 28672 data: 4 4 5 6 6 7 7
88682024-02-07T18:58:25.349Z30/30 Write block 561 len 16384 data: 5 4 4 7
88692024-02-07T18:58:25.349Z----------------------------------------------------------------
88702024-02-07T18:58:25.349Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:24
88712024-02-07T18:58:25.350ZStates: Active Active Active
88722024-02-07T18:58:25.350ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
88732024-02-07T18:58:25.350Z 7 Acked 1006 Write 8 Sent Done Done false
88742024-02-07T18:58:25.350Z 8 Acked 1007 Flush 0 Sent Done Done false
88752024-02-07T18:58:25.350Z 9 Acked 1008 Write 10 Sent Done Done false
88762024-02-07T18:58:25.350Z 10 Acked 1009 Read 8 Sent Done Done false
88772024-02-07T18:58:25.350Z 11 Acked 1010 Write 1 Sent Done Done false
88782024-02-07T18:58:25.350Z 12 Acked 1011 Read 2 Sent Done Done false
88792024-02-07T18:58:25.350Z 13 Acked 1012 Read 6 Sent Done Done false
88802024-02-07T18:58:25.350Z 14 Acked 1013 Write 2 Sent Done Done false
88812024-02-07T18:58:25.350Z 15 Acked 1014 Write 2 Sent Done Done false
88822024-02-07T18:58:25.350Z 16 Acked 1015 Write 3 Sent Done Done false
88832024-02-07T18:58:25.350Z 17 Acked 1016 Read 6 Sent Done Done false
88842024-02-07T18:58:25.350Z 18 Acked 1017 Read 10 Sent Done Done false
88852024-02-07T18:58:25.350Z 19 Acked 1018 Flush 0 Sent Done Done false
88862024-02-07T18:58:25.350Z 20 Acked 1019 Write 7 Sent Done Done false
88872024-02-07T18:58:25.350Z 21 Acked 1020 Read 1 Sent Done Done false
88882024-02-07T18:58:25.350Z 22 Acked 1021 Flush 0 Sent Done Done false
88892024-02-07T18:58:25.350Z 23 Acked 1022 Write 9 Sent Done Done false
88902024-02-07T18:58:25.350Z 24 Acked 1023 Read 7 Sent Done Done false
88912024-02-07T18:58:25.350Z 25 Acked 1024 Flush 0 Sent Done Done false
88922024-02-07T18:58:25.350Z 26 Acked 1025 Write 8 Sent Done Done false
88932024-02-07T18:58:25.351Z 27 Acked 1026 Write 10 Sent Done Done false
88942024-02-07T18:58:25.351Z 28 Acked 1027 Read 5 Sent Done Done false
88952024-02-07T18:58:25.351Z 29 Acked 1028 Write 7 Sent Sent Done false
88962024-02-07T18:58:25.351Z 30 Acked 1029 Write 4 Sent Sent Sent false
88972024-02-07T18:58:25.351Z STATES DS:0 DS:1 DS:2 TOTAL
88982024-02-07T18:58:25.351Z New 0 0 0 0
88992024-02-07T18:58:25.351Z Sent 24 2 1 27
89002024-02-07T18:58:25.351Z Done 0 22 23 45
89012024-02-07T18:58:25.351Z Skipped 0 0 0 0
89022024-02-07T18:58:25.351Z Error 0 0 0 0
89032024-02-07T18:58:25.351ZLast Flush: 1005 1024 1024
89042024-02-07T18:58:25.351ZDownstairs last five completed: 1005 1004 1003 1002 1001
89052024-02-07T18:58:25.351ZUpstairs last five completed: 30 29 28 27 26
89062024-02-07T18:58:25.351Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:24.570927864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1455}
89072024-02-07T18:58:25.352ZWrote out file "/tmp/test_repair_verify.data"
89082024-02-07T18:58:25.352Z
89092024-02-07T18:58:25.352ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89102024-02-07T18:58:25.377ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89112024-02-07T18:58:25.378Z 0 000-019 77 80 80 123 131 131 F T T <---
89122024-02-07T18:58:25.378Z 1 020-039 77 80 80 123 131 131 F F F <---
89132024-02-07T18:58:25.378Z 2 040-059 77 77 77 123 123 123 F F F
89142024-02-07T18:58:25.378Z 3 060-079 76 80 80 122 131 131 F F F <---
89152024-02-07T18:58:25.378Z 4 080-099 78 78 78 124 124 124 F F F
89162024-02-07T18:58:25.378Z 5 100-119 78 78 78 126 126 126 F F F
89172024-02-07T18:58:25.378Z 6 120-139 79 79 79 128 128 128 F F F
89182024-02-07T18:58:25.379Z 7 140-159 71 71 71 113 113 113 F T T <---
89192024-02-07T18:58:25.379Z 8 160-179 78 78 78 124 124 124 F F F
89202024-02-07T18:58:25.379Z 9 180-199 75 75 75 121 121 121 F F F
89212024-02-07T18:58:25.379Z 10 200-219 74 80 80 120 131 131 F F F <---
89222024-02-07T18:58:25.379Z 11 220-239 78 80 80 124 131 131 F F F <---
89232024-02-07T18:58:25.379Z 12 240-259 78 78 78 125 125 125 F F F
89242024-02-07T18:58:25.379Z 13 260-279 78 80 80 124 131 131 F F F <---
89252024-02-07T18:58:25.379Z 14 280-299 78 80 80 126 131 131 F F F <---
89262024-02-07T18:58:25.379Z 15 300-319 80 80 80 129 129 129 F F F
89272024-02-07T18:58:25.379Z 16 320-339 78 78 78 124 124 124 F T T <---
89282024-02-07T18:58:25.379Z 17 340-359 78 78 78 127 127 127 F F F
89292024-02-07T18:58:25.379Z 18 360-379 78 78 78 127 127 127 F F F
89302024-02-07T18:58:25.380Z 19 380-399 60 60 60 96 96 96 F F F
89312024-02-07T18:58:25.380Z 20 400-419 79 80 80 128 133 133 F F F <---
89322024-02-07T18:58:25.380Z 21 420-439 77 77 77 123 123 123 F F F
89332024-02-07T18:58:25.380Z 22 440-459 77 77 77 123 123 123 F F F
89342024-02-07T18:58:25.380Z 23 460-479 74 74 74 120 120 120 F F F
89352024-02-07T18:58:25.380Z 24 480-499 77 77 77 123 123 123 F F F
89362024-02-07T18:58:25.380Z 25 500-519 80 80 80 129 130 130 F F F <---
89372024-02-07T18:58:25.380Z 26 520-539 78 80 80 124 130 130 F F F <---
89382024-02-07T18:58:25.380Z 27 540-559 77 77 77 123 123 123 F F F
89392024-02-07T18:58:25.380Z 28 560-579 79 80 80 128 132 132 F T T <---
89402024-02-07T18:58:25.380Z 29 580-599 77 77 77 123 123 123 F F F
89412024-02-07T18:58:25.380ZMax gen: 80, Max flush: 133
89422024-02-07T18:58:25.380ZError: Difference in extent metadata found!
89432024-02-07T18:58:25.393ZA Difference in extent metadata is expected here
89442024-02-07T18:58:25.393ZOn loop 40
89452024-02-07T18:58:25.393Z
89462024-02-07T18:58:25.393ZVerifying data now
89472024-02-07T18:58:30.499ZNo change after verify
89482024-02-07T18:58:30.499ZLoop: 40 Downstairs dump after verify (and repair):
89492024-02-07T18:58:30.510ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89502024-02-07T18:58:30.510Z 0 000-019 81 81 81 134 134 134 F F F
89512024-02-07T18:58:30.510Z 1 020-039 80 80 80 131 131 131 F F F
89522024-02-07T18:58:30.510Z 2 040-059 77 77 77 123 123 123 F F F
89532024-02-07T18:58:30.510Z 3 060-079 80 80 80 131 131 131 F F F
89542024-02-07T18:58:30.510Z 4 080-099 78 78 78 124 124 124 F F F
89552024-02-07T18:58:30.510Z 5 100-119 78 78 78 126 126 126 F F F
89562024-02-07T18:58:30.510Z 6 120-139 79 79 79 128 128 128 F F F
89572024-02-07T18:58:30.510Z 7 140-159 81 81 81 134 134 134 F F F
89582024-02-07T18:58:30.510Z 8 160-179 78 78 78 124 124 124 F F F
89592024-02-07T18:58:30.510Z 9 180-199 75 75 75 121 121 121 F F F
89602024-02-07T18:58:30.510Z 10 200-219 80 80 80 131 131 131 F F F
89612024-02-07T18:58:30.510Z 11 220-239 80 80 80 131 131 131 F F F
89622024-02-07T18:58:30.510Z 12 240-259 78 78 78 125 125 125 F F F
89632024-02-07T18:58:30.510Z 13 260-279 80 80 80 131 131 131 F F F
89642024-02-07T18:58:30.511Z 14 280-299 80 80 80 131 131 131 F F F
89652024-02-07T18:58:30.511Z 15 300-319 80 80 80 129 129 129 F F F
89662024-02-07T18:58:30.511Z 16 320-339 81 81 81 134 134 134 F F F
89672024-02-07T18:58:30.511Z 17 340-359 78 78 78 127 127 127 F F F
89682024-02-07T18:58:30.511Z 18 360-379 78 78 78 127 127 127 F F F
89692024-02-07T18:58:30.511Z 19 380-399 60 60 60 96 96 96 F F F
89702024-02-07T18:58:30.511Z 20 400-419 80 80 80 133 133 133 F F F
89712024-02-07T18:58:30.511Z 21 420-439 77 77 77 123 123 123 F F F
89722024-02-07T18:58:30.511Z 22 440-459 77 77 77 123 123 123 F F F
89732024-02-07T18:58:30.511Z 23 460-479 74 74 74 120 120 120 F F F
89742024-02-07T18:58:30.511Z 24 480-499 77 77 77 123 123 123 F F F
89752024-02-07T18:58:30.511Z 25 500-519 80 80 80 130 130 130 F F F
89762024-02-07T18:58:30.511Z 26 520-539 80 80 80 130 130 130 F F F
89772024-02-07T18:58:30.511Z 27 540-559 77 77 77 123 123 123 F F F
89782024-02-07T18:58:30.511Z 28 560-579 81 81 81 134 134 134 F F F
89792024-02-07T18:58:30.511Z 29 580-599 77 77 77 123 123 123 F F F
89802024-02-07T18:58:30.511ZMax gen: 81, Max flush: 134
89812024-02-07T18:58:30.511Z
89822024-02-07T18:58:30.511ZBegin loop 41
89832024-02-07T18:58:30.512ZDownstairs to restart: 1
89842024-02-07T18:58:30.512Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270398418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463}
89852024-02-07T18:58:30.512Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
89862024-02-07T18:58:30.512Z,"time":"2024-02-07T18:58:29.27067988Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463}
89872024-02-07T18:58:30.512Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270735883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463}
89882024-02-07T18:58:30.512Z{"msg":"Crucible 7a90a21d-98b3-410b-9782-bdadc9c6f8ed has session id: 9692b1af-e7a1-41b4-9728-9b21ac78cdae","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270787276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89892024-02-07T18:58:30.512Z{"msg":"Upstairs opts: Upstairs UUID: 7a90a21d-98b3-410b-9782-bdadc9c6f8ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270821871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89902024-02-07T18:58:30.512Z{"msg":"Crucible stats registered with UUID: 7a90a21d-98b3-410b-9782-bdadc9c6f8ed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270847678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89912024-02-07T18:58:30.512Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270861576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463}
89922024-02-07T18:58:30.512Z{"msg":"7a90a21d-98b3-410b-9782-bdadc9c6f8ed active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270933406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89932024-02-07T18:58:30.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.270960902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89942024-02-07T18:58:30.512Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.27097928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89952024-02-07T18:58:30.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271014405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89962024-02-07T18:58:30.512Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271043741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89972024-02-07T18:58:30.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271091315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89982024-02-07T18:58:30.512Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.27112935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
89992024-02-07T18:58:30.512Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271176753Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90002024-02-07T18:58:30.512Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271207509Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90012024-02-07T18:58:30.512Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271307366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90022024-02-07T18:58:30.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271392364Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90032024-02-07T18:58:30.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.2714239Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90042024-02-07T18:58:30.512Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:29.271460205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90052024-02-07T18:58:30.512Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:29.271479412Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90062024-02-07T18:58:30.513Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:29.271514467Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90072024-02-07T18:58:30.513Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271529585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90082024-02-07T18:58:30.513Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271550583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90092024-02-07T18:58:30.513Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.271684055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90102024-02-07T18:58:30.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.272074112Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90112024-02-07T18:58:30.513Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.272109237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90122024-02-07T18:58:30.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.272124765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90132024-02-07T18:58:30.513Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.27215798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90142024-02-07T18:58:30.513Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.272741961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90152024-02-07T18:58:30.513Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.272987288Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90162024-02-07T18:58:30.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.27334869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90172024-02-07T18:58:30.513Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.273391424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90182024-02-07T18:58:30.513Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.273439347Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90192024-02-07T18:58:30.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.273509608Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90202024-02-07T18:58:30.513Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.273586627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90212024-02-07T18:58:30.513Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:29.273627632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90222024-02-07T18:58:31.040Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.272240409Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90232024-02-07T18:58:31.040Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.272336646Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90242024-02-07T18:58:31.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.273143307Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90252024-02-07T18:58:31.071Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.27318819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90262024-02-07T18:58:31.071Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.273663036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90272024-02-07T18:58:31.071Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274279013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90282024-02-07T18:58:31.072Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274313868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90292024-02-07T18:58:31.072Z{"msg":"[0]R flush_numbers[0..12]: [134, 131, 123, 131, 124, 126, 128, 134, 124, 121, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274334426Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90302024-02-07T18:58:31.072Z{"msg":"[0]R generation[0..12]: [81, 80, 77, 80, 78, 78, 79, 81, 78, 75, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274352463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90312024-02-07T18:58:31.072Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274380429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90322024-02-07T18:58:31.072Z{"msg":"[1]R flush_numbers[0..12]: [134, 131, 123, 131, 124, 126, 128, 134, 124, 121, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274398957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90332024-02-07T18:58:31.072Z{"msg":"[1]R generation[0..12]: [81, 80, 77, 80, 78, 78, 79, 81, 78, 75, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274425353Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90342024-02-07T18:58:31.073Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274444161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90352024-02-07T18:58:31.073Z{"msg":"[2]R flush_numbers[0..12]: [134, 131, 123, 131, 124, 126, 128, 134, 124, 121, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274470567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90362024-02-07T18:58:31.073Z{"msg":"[2]R generation[0..12]: [81, 80, 77, 80, 78, 78, 79, 81, 78, 75, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274488235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90372024-02-07T18:58:31.073Z{"msg":"Wait for a query_work_queue command to finish before sending IO
90382024-02-07T18:58:31.073Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274517221Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90392024-02-07T18:58:31.073Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274544627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90402024-02-07T18:58:31.073Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274571553Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90412024-02-07T18:58:31.073Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274588421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90422024-02-07T18:58:31.073Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274619287Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90432024-02-07T18:58:31.074Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274636885Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90442024-02-07T18:58:31.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274678599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90452024-02-07T18:58:31.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274693067Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90462024-02-07T18:58:31.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274706415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90472024-02-07T18:58:31.074Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274731242Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90482024-02-07T18:58:31.074Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.27474438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90492024-02-07T18:58:31.074Z{"msg":"7a90a21d-98b3-410b-9782-bdadc9c6f8ed is now active with session: 9692b1af-e7a1-41b4-9728-9b21ac78cdae","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274758918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90502024-02-07T18:58:31.074Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274779705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463,"session_id":"9692b1af-e7a1-41b4-9728-9b21ac78cdae"}
90512024-02-07T18:58:31.074Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:30.274792913Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463}
90522024-02-07T18:58:31.074ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90532024-02-07T18:58:31.075ZLoading write count information from file "/tmp/test_repair_verify.data"
90542024-02-07T18:58:31.075ZRun Repair workload
90552024-02-07T18:58:31.075Z01/30 Write block 575 len 32768 data: 5 4 4 5 5 5 7 7
90562024-02-07T18:58:31.075Z02/30 Read block 365 len 16384
90572024-02-07T18:58:31.075Z03/30 Read block 352 len 24576
90582024-02-07T18:58:31.075Z04/30 Read block 541 len 12288
90592024-02-07T18:58:31.075Z05/30 Read block 239 len 20480
90602024-02-07T18:58:31.075Z06/30 Write block 267 len 12288 data: 5 5 5
90612024-02-07T18:58:31.076Z07/30 Write block 161 len 16384 data: 6 8 9 9
90622024-02-07T18:58:31.076Z08/30 Write block 398 len 36864 data: 5 4 5 4 6 4 4 3 5
90632024-02-07T18:58:31.076Z09/30 Write block 242 len 32768 data: 15 13 12 8 8 7 6 4
90642024-02-07T18:58:31.076Z10/30 Read block 388 len 20480
90652024-02-07T18:58:31.090Z11/30 Read block 455 len 4096
90662024-02-07T18:58:31.090Z12/30 Write block 164 len 8192 data: 10 9
90672024-02-07T18:58:31.090Z13/30 Write block 413 len 4096 data: 11
90682024-02-07T18:58:31.090Z14/30 Read block 312 len 4096
90692024-02-07T18:58:31.091Z15/30 Read block 218 len 24576
90702024-02-07T18:58:31.091Z16/30 Read block 44 len 8192
90712024-02-07T18:58:31.091Z17/30 Read block 420 len 16384
90722024-02-07T18:58:31.091Z18/30 Read block 331 len 36864
90732024-02-07T18:58:31.091Z19/30 Write block 372 len 16384 data: 4 4 3 3
90742024-02-07T18:58:31.091Z20/30 Write block 20 len 16384 data: 12 11 11 11
90752024-02-07T18:58:31.091Z21/30 Flush
90762024-02-07T18:58:31.091Z22/30 Write block 479 len 8192 data: 4 4
90772024-02-07T18:58:31.092Z23/30 Read block 485 len 28672
90782024-02-07T18:58:31.092Z24/30 Write block 477 len 32768 data: 5 5 5 5 4 4 6 5
90792024-02-07T18:58:31.092Z25/30 Write block 267 len 36864 data: 6 6 6 4 5 5 5 4 4
90802024-02-07T18:58:31.092Z26/30 Read block 414 len 8192
90812024-02-07T18:58:31.092Z27/30 Write block 239 len 8192 data: 12 14
90822024-02-07T18:58:31.092Z28/30 Read block 243 len 28672
90832024-02-07T18:58:31.092Z29/30 Read block 560 len 4096
90842024-02-07T18:58:31.092Z30/30 Write block 103 len 36864 data: 5 8 9 8 8 8 9 8 8
90852024-02-07T18:58:31.093Z----------------------------------------------------------------
90862024-02-07T18:58:31.093Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
90872024-02-07T18:58:31.093ZStates: Active Active Active
90882024-02-07T18:58:31.093ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
90892024-02-07T18:58:31.093Z 1 Acked 1000 Write 8 Done Sent Done false
90902024-02-07T18:58:31.093Z 2 Acked 1001 Read 4 Done Sent Done false
90912024-02-07T18:58:31.093Z 3 Acked 1002 Read 6 Done Sent Done false
90922024-02-07T18:58:31.093Z 4 Acked 1003 Read 3 Done Sent Done false
90932024-02-07T18:58:31.093Z 5 Acked 1004 Read 5 Done Sent Done false
90942024-02-07T18:58:31.094Z 6 Acked 1005 Write 3 Done Sent Done false
90952024-02-07T18:58:31.094Z 7 Acked 1006 Write 4 Done Sent Done false
90962024-02-07T18:58:31.094Z 8 Acked 1007 Write 9 Done Sent Done false
90972024-02-07T18:58:31.094Z 9 Acked 1008 Write 8 Done Sent Done false
90982024-02-07T18:58:31.094Z 10 Acked 1009 Read 5 Done Sent Done false
90992024-02-07T18:58:31.094Z 11 Acked 1010 Read 1 Done Sent Done false
91002024-02-07T18:58:31.094Z 12 Acked 1011 Write 2 Done Sent Done false
91012024-02-07T18:58:31.094Z 13 Acked 1012 Write 1 Done Sent Done false
91022024-02-07T18:58:31.095Z 14 Acked 1013 Read 1 Done Sent Done false
91032024-02-07T18:58:31.095Z 15 Acked 1014 Read 6 Done Sent Done false
91042024-02-07T18:58:31.095Z 16 Acked 1015 Read 2 Done Sent Done false
91052024-02-07T18:58:31.095Z 17 Acked 1016 Read 4 Done Sent Done false
91062024-02-07T18:58:31.095Z 18 Acked 1017 Read 9 Done Sent Done false
91072024-02-07T18:58:31.095Z 19 Acked 1018 Write 4 Done Sent Done false
91082024-02-07T18:58:31.095Z 20 Acked 1019 Write 4 Done Sent Done false
91092024-02-07T18:58:31.095Z 21 Acked 1020 Flush 0 Done Sent Done false
91102024-02-07T18:58:31.096Z 22 Acked 1021 Write 2 Done Sent Done false
91112024-02-07T18:58:31.096Z 23 Acked 1022 Read 7 Done Sent Done false
91122024-02-07T18:58:31.096Z 24 Acked 1023 Write 8 Done Sent Done false
91132024-02-07T18:58:31.096Z 25 Acked 1024 Write 9 Done Sent Done false
91142024-02-07T18:58:31.096Z 26 Acked 1025 Read 2 Done Sent Done false
91152024-02-07T18:58:31.096Z 27 Acked 1026 Write 2 Done Sent Done false
91162024-02-07T18:58:31.096Z 28 Acked 1027 Read 7 Done Sent Done false
91172024-02-07T18:58:31.097Z 29 Acked 1028 Read 1 Done Sent Done false
91182024-02-07T18:58:31.097Z 30 Acked 1029 Write 9 Sent Sent Sent false
91192024-02-07T18:58:31.097Z STATES DS:0 DS:1 DS:2 TOTAL
91202024-02-07T18:58:31.097Z New 0 0 0 0
91212024-02-07T18:58:31.097Z Sent 1 30 1 32
91222024-02-07T18:58:31.097Z Done 29 0 29 58
91232024-02-07T18:58:31.097Z Skipped 0 0 0 0
91242024-02-07T18:58:31.097Z Error 0 0 0 0
91252024-02-07T18:58:31.098ZLast Flush: 1020 0 1020
91262024-02-07T18:58:31.098ZDownstairs last five completed:
91272024-02-07T18:58:31.098ZUpstairs last five completed: 30 29 28 27 26
91282024-02-07T18:58:31.098Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:30.303586639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1463}
91292024-02-07T18:58:31.098ZWrote out file "/tmp/test_repair_verify.data"
91302024-02-07T18:58:31.098Z
91312024-02-07T18:58:31.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91322024-02-07T18:58:31.532ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91332024-02-07T18:58:31.532Z 0 000-019 81 81 81 134 134 134 F F F
91342024-02-07T18:58:31.532Z 1 020-039 82 80 82 135 131 135 F F F <---
91352024-02-07T18:58:31.532Z 2 040-059 77 77 77 123 123 123 F F F
91362024-02-07T18:58:31.533Z 3 060-079 80 80 80 131 131 131 F F F
91372024-02-07T18:58:31.533Z 4 080-099 78 78 78 124 124 124 F F F
91382024-02-07T18:58:31.533Z 5 100-119 78 78 78 126 126 126 T F T <---
91392024-02-07T18:58:31.533Z 6 120-139 79 79 79 128 128 128 F F F
91402024-02-07T18:58:31.533Z 7 140-159 81 81 81 134 134 134 F F F
91412024-02-07T18:58:31.533Z 8 160-179 82 78 82 135 124 135 F F F <---
91422024-02-07T18:58:31.534Z 9 180-199 75 75 75 121 121 121 F F F
91432024-02-07T18:58:31.534Z 10 200-219 80 80 80 131 131 131 F F F
91442024-02-07T18:58:31.534Z 11 220-239 80 80 80 131 131 131 T F T <---
91452024-02-07T18:58:31.534Z 12 240-259 82 78 82 135 125 135 T F T <---
91462024-02-07T18:58:31.534Z 13 260-279 82 80 82 135 131 135 T F T <---
91472024-02-07T18:58:31.534Z 14 280-299 80 80 80 131 131 131 F F F
91482024-02-07T18:58:31.534Z 15 300-319 80 80 80 129 129 129 F F F
91492024-02-07T18:58:31.535Z 16 320-339 81 81 81 134 134 134 F F F
91502024-02-07T18:58:31.535Z 17 340-359 78 78 78 127 127 127 F F F
91512024-02-07T18:58:31.535Z 18 360-379 82 78 82 135 127 135 F F F <---
91522024-02-07T18:58:31.535Z 19 380-399 82 60 82 135 96 135 F F F <---
91532024-02-07T18:58:31.535Z 20 400-419 82 80 82 135 133 135 F F F <---
91542024-02-07T18:58:31.535Z 21 420-439 77 77 77 123 123 123 F F F
91552024-02-07T18:58:31.536Z 22 440-459 77 77 77 123 123 123 F F F
91562024-02-07T18:58:31.536Z 23 460-479 74 74 74 120 120 120 T F T <---
91572024-02-07T18:58:31.536Z 24 480-499 77 77 77 123 123 123 T F T <---
91582024-02-07T18:58:31.536Z 25 500-519 80 80 80 130 130 130 F F F
91592024-02-07T18:58:31.536Z 26 520-539 80 80 80 130 130 130 F F F
91602024-02-07T18:58:31.536Z 27 540-559 77 77 77 123 123 123 F F F
91612024-02-07T18:58:31.537Z 28 560-579 82 81 82 135 134 135 F F F <---
91622024-02-07T18:58:31.537Z 29 580-599 82 77 82 135 123 135 F F F <---
91632024-02-07T18:58:31.537ZMax gen: 82, Max flush: 135
91642024-02-07T18:58:31.537ZError: Difference in extent metadata found!
91652024-02-07T18:58:31.537ZA Difference in extent metadata is expected here
91662024-02-07T18:58:31.537ZOn loop 41
91672024-02-07T18:58:31.537Z
91682024-02-07T18:58:31.538ZVerifying data now
91692024-02-07T18:58:35.749ZNo change after verify
91702024-02-07T18:58:35.749ZLoop: 41 Downstairs dump after verify (and repair):
91712024-02-07T18:58:35.772ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91722024-02-07T18:58:35.772Z 0 000-019 81 81 81 134 134 134 F F F
91732024-02-07T18:58:35.773Z 1 020-039 82 82 82 135 135 135 F F F
91742024-02-07T18:58:35.773Z 2 040-059 77 77 77 123 123 123 F F F
91752024-02-07T18:58:35.773Z 3 060-079 80 80 80 131 131 131 F F F
91762024-02-07T18:58:35.773Z 4 080-099 78 78 78 124 124 124 F F F
91772024-02-07T18:58:35.774Z 5 100-119 83 83 83 136 136 136 F F F
91782024-02-07T18:58:35.774Z 6 120-139 79 79 79 128 128 128 F F F
91792024-02-07T18:58:35.774Z 7 140-159 81 81 81 134 134 134 F F F
91802024-02-07T18:58:35.775Z 8 160-179 82 82 82 135 135 135 F F F
91812024-02-07T18:58:35.775Z 9 180-199 75 75 75 121 121 121 F F F
91822024-02-07T18:58:35.775Z 10 200-219 80 80 80 131 131 131 F F F
91832024-02-07T18:58:35.775Z 11 220-239 83 83 83 136 136 136 F F F
91842024-02-07T18:58:35.775Z 12 240-259 83 83 83 136 136 136 F F F
91852024-02-07T18:58:35.775Z 13 260-279 83 83 83 136 136 136 F F F
91862024-02-07T18:58:35.776Z 14 280-299 80 80 80 131 131 131 F F F
91872024-02-07T18:58:35.776Z 15 300-319 80 80 80 129 129 129 F F F
91882024-02-07T18:58:35.776Z 16 320-339 81 81 81 134 134 134 F F F
91892024-02-07T18:58:35.776Z 17 340-359 78 78 78 127 127 127 F F F
91902024-02-07T18:58:35.776Z 18 360-379 82 82 82 135 135 135 F F F
91912024-02-07T18:58:35.776Z 19 380-399 82 82 82 135 135 135 F F F
91922024-02-07T18:58:35.776Z 20 400-419 82 82 82 135 135 135 F F F
91932024-02-07T18:58:35.777Z 21 420-439 77 77 77 123 123 123 F F F
91942024-02-07T18:58:35.777Z 22 440-459 77 77 77 123 123 123 F F F
91952024-02-07T18:58:35.777Z 23 460-479 83 83 83 136 136 136 F F F
91962024-02-07T18:58:35.777Z 24 480-499 83 83 83 136 136 136 F F F
91972024-02-07T18:58:35.777Z 25 500-519 80 80 80 130 130 130 F F F
91982024-02-07T18:58:35.777Z 26 520-539 80 80 80 130 130 130 F F F
91992024-02-07T18:58:35.777Z 27 540-559 77 77 77 123 123 123 F F F
92002024-02-07T18:58:35.778Z 28 560-579 82 82 82 135 135 135 F F F
92012024-02-07T18:58:35.778Z 29 580-599 82 82 82 135 135 135 F F F
92022024-02-07T18:58:35.778ZMax gen: 83, Max flush: 136
92032024-02-07T18:58:35.779Z
92042024-02-07T18:58:35.779ZBegin loop 42
92052024-02-07T18:58:35.779ZDownstairs to restart: 1
92062024-02-07T18:58:35.795Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004237983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471}
92072024-02-07T18:58:35.795Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004535913Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471}
92082024-02-07T18:58:35.795Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004565269Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471}
92092024-02-07T18:58:35.796Z{"msg":"Crucible 4a0ee7a8-4155-4270-a9a3-c1bc849dc422 has session id: d3d2458e-4a09-42a1-98a2-bc7c79883288","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004606043Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal"Crucible runtime is spawned
92102024-02-07T18:58:35.796Z,"pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92112024-02-07T18:58:35.796Z{"msg":"Upstairs opts: Upstairs UUID: 4a0ee7a8-4155-4270-a9a3-c1bc849dc422, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004687382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92122024-02-07T18:58:35.796Z{"msg":"Crucible stats registered with UUID: 4a0ee7a8-4155-4270-a9a3-c1bc849dc422","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004734016Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92132024-02-07T18:58:35.796Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004763902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471}
92142024-02-07T18:58:35.797Z{"msg":"4a0ee7a8-4155-4270-a9a3-c1bc849dc422 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004792578Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92152024-02-07T18:58:35.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004809286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92162024-02-07T18:58:35.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004847361Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92172024-02-07T18:58:35.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004877557Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92182024-02-07T18:58:35.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004907622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92192024-02-07T18:58:35.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004936089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92202024-02-07T18:58:35.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004963805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92212024-02-07T18:58:35.798Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.004993441Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92222024-02-07T18:58:35.798Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.005024807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92232024-02-07T18:58:35.798Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.005053803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92242024-02-07T18:58:35.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.00521918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92252024-02-07T18:58:35.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.005279602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92262024-02-07T18:58:35.799Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:35.005352732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92272024-02-07T18:58:35.799Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:35.005429742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92282024-02-07T18:58:35.799Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:35.005471716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92292024-02-07T18:58:35.799Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.005508361Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92302024-02-07T18:58:35.799Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.005549616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92312024-02-07T18:58:35.799Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.00559552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92322024-02-07T18:58:35.800Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.006040789Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92332024-02-07T18:58:35.800Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.006077354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92342024-02-07T18:58:35.800Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.006094272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92352024-02-07T18:58:35.800Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.00611093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92362024-02-07T18:58:35.800Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.006657146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92372024-02-07T18:58:35.800Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.006693721Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92382024-02-07T18:58:35.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.007243566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92392024-02-07T18:58:35.801Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.007279322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92402024-02-07T18:58:35.802Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.00729446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92412024-02-07T18:58:35.803Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.007313287Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92422024-02-07T18:58:35.804Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.007338284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92432024-02-07T18:58:35.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:35.00736839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92442024-02-07T18:58:36.769Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.007081408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92452024-02-07T18:58:36.775Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.00721844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92462024-02-07T18:58:36.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.008077794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92472024-02-07T18:58:36.776Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.008116218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92482024-02-07T18:58:36.776Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.008674883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92492024-02-07T18:58:36.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009268813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92502024-02-07T18:58:36.776Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009302018Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92512024-02-07T18:58:36.777Z{"msg":"[0]R flush_numbers[0..12]: [134, 135, 123, 131, 124, 136, 128, 134, 135, 121, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009323135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92522024-02-07T18:58:36.777Z{"msg":"[0]R generation[0..12]: [81, 82, 77, 80, 78, 83, 79, 81, 82, 75, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009349411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92532024-02-07T18:58:36.777Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009367909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92542024-02-07T18:58:36.777Z{"msg":"[1]R flush_numbers[0..12]: [134, 135, 123, 131, 124, 136, 128, 134, 135, 121, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009394265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92552024-02-07T18:58:36.777Z{"msg":"[1]R generation[0..12]: [81, 82, 77, 80, 78, 83, 79, 81, 82, 75, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009412213Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92562024-02-07T18:58:36.778Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009431771Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92572024-02-07T18:58:36.778Z{"msg":"[2]R flush_numbers[0..12]: [134, 135, 123, 131, 124, 136, 128, 134, 135, 121, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009450448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92582024-02-07T18:58:36.778Z{"msg":"[2]R generation[0..12]: [81, 82, 77, 80, 78, 83, 79, 81, 82, 75, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009483664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92592024-02-07T18:58:36.778Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009502171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92602024-02-07T18:58:36.779ZWait for a query_work_queue command to finish before sending IO
92612024-02-07T18:58:36.779Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009532337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92622024-02-07T18:58:36.779Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009550315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92632024-02-07T18:58:36.779Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009575711Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92642024-02-07T18:58:36.780Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009592939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92652024-02-07T18:58:36.780Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009618915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92662024-02-07T18:58:36.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009634753Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92672024-02-07T18:58:36.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009668268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92682024-02-07T18:58:36.780Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009694895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92692024-02-07T18:58:36.781Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009712802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92702024-02-07T18:58:36.781Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009737879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92712024-02-07T18:58:36.781Z{"msg":"4a0ee7a8-4155-4270-a9a3-c1bc849dc422 is now active with session: d3d2458e-4a09-42a1-98a2-bc7c79883288","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009755157Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92722024-02-07T18:58:36.781Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009779224Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471,"session_id":"d3d2458e-4a09-42a1-98a2-bc7c79883288"}
92732024-02-07T18:58:36.781Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:36.009795171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471}
92742024-02-07T18:58:36.781ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92752024-02-07T18:58:36.781ZLoading write count information from file "/tmp/test_repair_verify.data"
92762024-02-07T18:58:36.782ZRun Repair workload
92772024-02-07T18:58:36.782Z01/30 Write block 263 len 16384 data: 9 9 8 6
92782024-02-07T18:58:36.793Z02/30 Write block 95 len 20480 data: 10 10 9 8 7
92792024-02-07T18:58:36.794Z03/30 Write block 420 len 4096 data: 5
92802024-02-07T18:58:36.794Z04/30 Flush
92812024-02-07T18:58:36.794Z05/30 Write block 139 len 36864 data: 5 5 9 9 10 10 9 11 12
92822024-02-07T18:58:36.795Z06/30 Read block 341 len 16384
92832024-02-07T18:58:36.795Z07/30 Write block 447 len 8192 data: 7 7
92842024-02-07T18:58:36.795Z08/30 Read block 104 len 32768
92852024-02-07T18:58:36.795Z09/30 Read block 136 len 32768
92862024-02-07T18:58:36.795Z10/30 Read block 442 len 32768
92872024-02-07T18:58:36.795Z11/30 Write block 579 len 16384 data: 6 6 8 8
92882024-02-07T18:58:36.796Z12/30 Read block 144 len 24576
92892024-02-07T18:58:36.796Z13/30 Read block 12 len 8192
92902024-02-07T18:58:36.796Z14/30 Write block 498 len 12288 data: 6 5 5
92912024-02-07T18:58:36.796Z15/30 Read block 388 len 16384
92922024-02-07T18:58:36.796Z16/30 Flush
92932024-02-07T18:58:36.823Z17/30 Write block 34 len 20480 data: 10 9 9 6 5
92942024-02-07T18:58:36.824Z18/30 Read block 315 len 16384
92952024-02-07T18:58:36.824Z19/30 Write block 567 len 4096 data: 7
92962024-02-07T18:58:36.824Z20/30 Read block 398 len 8192
92972024-02-07T18:58:36.824Z21/30 Write block 425 len 36864 data: 9 8 7 10 9 8 8 6 4
92982024-02-07T18:58:36.824Z22/30 Read block 475 len 4096
92992024-02-07T18:58:36.825Z23/30 Write block 107 len 28672 data: 9 9 10 9 9 6 5
93002024-02-07T18:58:36.825Z24/30 Read block 378 len 40960
93012024-02-07T18:58:36.825Z25/30 Write block 238 len 4096 data: 12
93022024-02-07T18:58:36.825Z26/30 Read block 260 len 32768
93032024-02-07T18:58:36.825Z27/30 Read block 509 len 32768
93042024-02-07T18:58:36.826Z28/30 Write block 380 len 40960 data: 4 5 5 5 4 4 6 6 7 8
93052024-02-07T18:58:36.826Z29/30 Read block 204 len 12288
93062024-02-07T18:58:36.826Z30/30 Read block 23 len 16384
93072024-02-07T18:58:36.826Z----------------------------------------------------------------
93082024-02-07T18:58:36.827Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
93092024-02-07T18:58:36.827ZStates: Active Active Active
93102024-02-07T18:58:36.827ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93112024-02-07T18:58:36.827Z 1 Acked 1000 Write 4 Done Sent Done false
93122024-02-07T18:58:36.827Z 2 Acked 1001 Write 5 Done Sent Done false
93132024-02-07T18:58:36.827Z 3 Acked 1002 Write 1 Done Sent Done false
93142024-02-07T18:58:36.828Z 4 Acked 1003 Flush 0 Done Sent Done false
93152024-02-07T18:58:36.828Z 5 Acked 1004 Write 9 Done Sent Done false
93162024-02-07T18:58:36.828Z 6 Acked 1005 Read 4 Done Sent Done false
93172024-02-07T18:58:36.828Z 7 Acked 1006 Write 2 Done Sent Done false
93182024-02-07T18:58:36.828Z 8 Acked 1007 Read 8 Done Sent Done false
93192024-02-07T18:58:36.829Z 9 Acked 1008 Read 8 Done Sent Done false
93202024-02-07T18:58:36.829Z 10 Acked 1009 Read 8 Done Sent Done false
93212024-02-07T18:58:36.829Z 11 Acked 1010 Write 4 Done Sent Done false
93222024-02-07T18:58:36.829Z 12 Acked 1011 Read 6 Done Sent Done false
93232024-02-07T18:58:36.829Z 13 Acked 1012 Read 2 Done Sent Done false
93242024-02-07T18:58:36.830Z 14 Acked 1013 Write 3 Done Sent Done false
93252024-02-07T18:58:36.830Z 15 Acked 1014 Read 4 Done Sent Done false
93262024-02-07T18:58:36.830Z 16 Acked 1015 Flush 0 Done Sent Done false
93272024-02-07T18:58:36.831Z 17 Acked 1016 Write 5 Done Sent Done false
93282024-02-07T18:58:36.831Z 18 Acked 1017 Read 4 Done Sent Done false
93292024-02-07T18:58:36.831Z 19 Acked 1018 Write 1 Done Sent Done false
93302024-02-07T18:58:36.831Z 20 Acked 1019 Read 2 Done Sent Done false
93312024-02-07T18:58:36.831Z 21 Acked 1020 Write 9 Done Sent Done false
93322024-02-07T18:58:36.832Z 22 Acked 1021 Read 1 Done Sent Done false
93332024-02-07T18:58:36.832Z 23 Acked 1022 Write 7 Done Sent Done false
93342024-02-07T18:58:36.832Z 24 Acked 1023 Read 10 Done Sent Done false
93352024-02-07T18:58:36.833Z 25 Acked 1024 Write 1 Done Sent Done false
93362024-02-07T18:58:36.833Z 26 Acked 1025 Read 8 Done Sent Done false
93372024-02-07T18:58:36.833Z 27 Acked 1026 Read 8 Done Sent Done false
93382024-02-07T18:58:36.833Z 28 Acked 1027 Write 10 Done Sent Done false
93392024-02-07T18:58:36.834Z 29 Acked 1028 Read 3 Done Sent Done false
93402024-02-07T18:58:36.834Z 30 Acked 1029 Read 4 Sent Sent Done false
93412024-02-07T18:58:36.834Z STATES DS:0 DS:1 DS:2 TOTAL
93422024-02-07T18:58:36.834Z New 0 0 0 0
93432024-02-07T18:58:36.835Z Sent 1 30 0 31
93442024-02-07T18:58:36.835Z Done 29 0 30 59
93452024-02-07T18:58:36.835Z Skipped 0 0 0 0
93462024-02-07T18:58:36.836Z Error 0 0 0 0
93472024-02-07T18:58:36.836ZLast Flush: 1015 0 1015
93482024-02-07T18:58:36.836ZDownstairs last five completed:
93492024-02-07T18:58:36.836ZUpstairs last five completed: 30 29 28 27 26
93502024-02-07T18:58:36.837Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:36.039658212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1471}
93512024-02-07T18:58:36.837ZWrote out file "/tmp/test_repair_verify.data"
93522024-02-07T18:58:36.837Z
93532024-02-07T18:58:36.843ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93542024-02-07T18:58:36.844ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93552024-02-07T18:58:36.844Z 0 000-019 81 81 81 134 134 134 F F F
93562024-02-07T18:58:36.844Z 1 020-039 82 82 82 135 135 135 T F T <---
93572024-02-07T18:58:36.844Z 2 040-059 77 77 77 123 123 123 F F F
93582024-02-07T18:58:36.844Z 3 060-079 80 80 80 131 131 131 F F F
93592024-02-07T18:58:36.845Z 4 080-099 84 78 84 137 124 137 F F F <---
93602024-02-07T18:58:36.845Z 5 100-119 83 83 83 136 136 136 T F T <---
93612024-02-07T18:58:36.845Z 6 120-139 84 79 84 138 128 138 F F F <---
93622024-02-07T18:58:36.845Z 7 140-159 84 81 84 138 134 138 F F F <---
93632024-02-07T18:58:36.845Z 8 160-179 82 82 82 135 135 135 F F F
93642024-02-07T18:58:36.845Z 9 180-199 75 75 75 121 121 121 F F F
93652024-02-07T18:58:36.846Z 10 200-219 80 80 80 131 131 131 F F F
93662024-02-07T18:58:36.846Z 11 220-239 83 83 83 136 136 136 T F T <---
93672024-02-07T18:58:36.846Z 12 240-259 83 83 83 136 136 136 F F F
93682024-02-07T18:58:36.846Z 13 260-279 84 83 84 137 136 137 F F F <---
93692024-02-07T18:58:36.846Z 14 280-299 80 80 80 131 131 131 F F F
93702024-02-07T18:58:36.846Z 15 300-319 80 80 80 129 129 129 F F F
93712024-02-07T18:58:36.847Z 16 320-339 81 81 81 134 134 134 F F F
93722024-02-07T18:58:36.847Z 17 340-359 78 78 78 127 127 127 F F F
93732024-02-07T18:58:36.847Z 18 360-379 82 82 82 135 135 135 F F F
93742024-02-07T18:58:36.847Z 19 380-399 82 82 82 135 135 135 T F T <---
93752024-02-07T18:58:36.848Z 20 400-419 82 82 82 135 135 135 F F F
93762024-02-07T18:58:36.848Z 21 420-439 84 77 84 137 123 137 T F T <---
93772024-02-07T18:58:36.848Z 22 440-459 84 77 84 138 123 138 F F F <---
93782024-02-07T18:58:36.848Z 23 460-479 83 83 83 136 136 136 F F F
93792024-02-07T18:58:36.848Z 24 480-499 84 83 84 138 136 138 F F F <---
93802024-02-07T18:58:36.848Z 25 500-519 84 80 84 138 130 138 F F F <---
93812024-02-07T18:58:36.849Z 26 520-539 80 80 80 130 130 130 F F F
93822024-02-07T18:58:36.849Z 27 540-559 77 77 77 123 123 123 F F F
93832024-02-07T18:58:36.849Z 28 560-579 84 82 84 138 135 138 T F T <---
93842024-02-07T18:58:36.849Z 29 580-599 84 82 84 138 135 138 F F F <---
93852024-02-07T18:58:36.849ZMax gen: 84, Max flush: 138
93862024-02-07T18:58:36.850ZError: Difference in extent metadata found!
93872024-02-07T18:58:36.850ZA Difference in extent metadata is expected here
93882024-02-07T18:58:36.850ZOn loop 42
93892024-02-07T18:58:36.850Z
93902024-02-07T18:58:36.850ZVerifying data now
93912024-02-07T18:58:41.512ZNo change after verify
93922024-02-07T18:58:41.512ZLoop: 42 Downstairs dump after verify (and repair):
93932024-02-07T18:58:41.526ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93942024-02-07T18:58:41.526Z 0 000-019 81 81 81 134 134 134 F F F
93952024-02-07T18:58:41.527Z 1 020-039 85 85 85 139 139 139 F F F
93962024-02-07T18:58:41.527Z 2 040-059 77 77 77 123 123 123 F F F
93972024-02-07T18:58:41.527Z 3 060-079 80 80 80 131 131 131 F F F
93982024-02-07T18:58:41.527Z 4 080-099 84 84 84 137 137 137 F F F
93992024-02-07T18:58:41.527Z 5 100-119 85 85 85 139 139 139 F F F
94002024-02-07T18:58:41.527Z 6 120-139 84 84 84 138 138 138 F F F
94012024-02-07T18:58:41.527Z 7 140-159 84 84 84 138 138 138 F F F
94022024-02-07T18:58:41.527Z 8 160-179 82 82 82 135 135 135 F F F
94032024-02-07T18:58:41.527Z 9 180-199 75 75 75 121 121 121 F F F
94042024-02-07T18:58:41.532Z 10 200-219 80 80 80 131 131 131 F F F
94052024-02-07T18:58:41.532Z 11 220-239 85 85 85 139 139 139 F F F
94062024-02-07T18:58:41.532Z 12 240-259 83 83 83 136 136 136 F F F
94072024-02-07T18:58:41.532Z 13 260-279 84 84 84 137 137 137 F F F
94082024-02-07T18:58:41.532Z 14 280-299 80 80 80 131 131 131 F F F
94092024-02-07T18:58:41.532Z 15 300-319 80 80 80 129 129 129 F F F
94102024-02-07T18:58:41.532Z 16 320-339 81 81 81 134 134 134 F F F
94112024-02-07T18:58:41.532Z 17 340-359 78 78 78 127 127 127 F F F
94122024-02-07T18:58:41.533Z 18 360-379 82 82 82 135 135 135 F F F
94132024-02-07T18:58:41.533Z 19 380-399 85 85 85 139 139 139 F F F
94142024-02-07T18:58:41.533Z 20 400-419 82 82 82 135 135 135 F F F
94152024-02-07T18:58:41.533Z 21 420-439 85 85 85 139 139 139 F F F
94162024-02-07T18:58:41.533Z 22 440-459 84 84 84 138 138 138 F F F
94172024-02-07T18:58:41.533Z 23 460-479 83 83 83 136 136 136 F F F
94182024-02-07T18:58:41.533Z 24 480-499 84 84 84 138 138 138 F F F
94192024-02-07T18:58:41.533Z 25 500-519 84 84 84 138 138 138 F F F
94202024-02-07T18:58:41.533Z 26 520-539 80 80 80 130 130 130 F F F
94212024-02-07T18:58:41.533Z 27 540-559 77 77 77 123 123 123 F F F
94222024-02-07T18:58:41.534Z 28 560-579 85 85 85 139 139 139 F F F
94232024-02-07T18:58:41.534Z 29 580-599 84 84 84 138 138 138 F F F
94242024-02-07T18:58:41.534ZMax gen: 85, Max flush: 139
94252024-02-07T18:58:41.534Z
94262024-02-07T18:58:41.534ZBegin loop 43
94272024-02-07T18:58:41.534ZDownstairs to restart: 1
94282024-02-07T18:58:41.549Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786227615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479}
94292024-02-07T18:58:41.549Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786474911Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479}
94302024-02-07T18:58:41.549Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786492159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479}
94312024-02-07T18:58:41.556Z{"msg":"Crucible 9cf049db-b96c-4db0-8857-909ea811979a has session id: 3a98678b-cc1b-4b5a-bb82-a351cb5658fa","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786514806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94322024-02-07T18:58:41.556Z{"msg":"Upstairs opts: Upstairs UUID: 9cf049db-b96c-4db0-8857-909ea811979a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
94332024-02-07T18:58:41.556Z,"time":"2024-02-07T18:58:40.786554041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94342024-02-07T18:58:41.556Z{"msg":"Crucible stats registered with UUID: 9cf049db-b96c-4db0-8857-909ea811979a","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786587676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94352024-02-07T18:58:41.556Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.78663Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479}
94362024-02-07T18:58:41.556Z{"msg":"9cf049db-b96c-4db0-8857-909ea811979a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786807406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94372024-02-07T18:58:41.556Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786838602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94382024-02-07T18:58:41.556Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786865808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94392024-02-07T18:58:41.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786905053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94402024-02-07T18:58:41.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786934999Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94412024-02-07T18:58:41.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786959036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94422024-02-07T18:58:41.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.786984212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94432024-02-07T18:58:41.557Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787042695Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94442024-02-07T18:58:41.557Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.78707549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94452024-02-07T18:58:41.557Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787112625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94462024-02-07T18:58:41.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787138591Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94472024-02-07T18:58:41.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787180766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94482024-02-07T18:58:41.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.78722611Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94492024-02-07T18:58:41.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787844816Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94502024-02-07T18:58:41.558Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.7878914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94512024-02-07T18:58:41.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787916546Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94522024-02-07T18:58:41.558Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.787956151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94532024-02-07T18:58:41.558Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.788485409Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94542024-02-07T18:58:41.558Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.788531023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94552024-02-07T18:58:41.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789128482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94562024-02-07T18:58:41.558Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789189924Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94572024-02-07T18:58:41.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789237418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94582024-02-07T18:58:41.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789256335Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94592024-02-07T18:58:41.558Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789272653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94602024-02-07T18:58:41.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789286841Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94612024-02-07T18:58:41.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789314127Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94622024-02-07T18:58:41.559Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789330785Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94632024-02-07T18:58:41.559Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.789862303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94642024-02-07T18:58:41.559Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790314442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94652024-02-07T18:58:41.559Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790360876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94662024-02-07T18:58:41.559Z{"msg":"[0]R flush_numbers[0..12]: [134, 139, 123, 131, 137, 139, 138, 138, 135, 121, 131, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790380803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94672024-02-07T18:58:41.559Z{"msg":"[0]R generation[0..12]: [81, 85, 77, 80, 84, 85, 84, 84, 82, 75, 80, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790396291Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94682024-02-07T18:58:41.565Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790411089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94692024-02-07T18:58:41.565Z{"msg":"[1]R flush_numbers[0..12]: [134, 139, 123, 131, 137, 139, 138, 138, 135, 121, 131, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790435286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94702024-02-07T18:58:41.565Z{"msg":"[1]R generation[0..12]: [81, 85, 77, 80, 84, 85, 84, 84, 82, 75, 80, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790449864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94712024-02-07T18:58:41.565Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790470691Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94722024-02-07T18:58:41.565Z{"msg":"[2]R flush_numbers[0..12]: [134, 139, 123, 131, 137, 139, 138, 138, 135, 121, 131, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790485489Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94732024-02-07T18:58:41.565Z{"msg":"[2]R generation[0..12]: [81, 85, 77, 80, 84, 85, 84, 84, 82, 75, 80, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790505886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94742024-02-07T18:58:41.565Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790520324Z","Wait for a query_work_queue command to finish before sending IO
94752024-02-07T18:58:41.566Zhostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94762024-02-07T18:58:41.566Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.79055125Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94772024-02-07T18:58:41.566Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790565738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94782024-02-07T18:58:41.566Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790578996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94792024-02-07T18:58:41.566Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790600063Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94802024-02-07T18:58:41.566Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790614332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94812024-02-07T18:58:41.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790632849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94822024-02-07T18:58:41.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790646607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94832024-02-07T18:58:41.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790666185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94842024-02-07T18:58:41.567Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790680103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94852024-02-07T18:58:41.567Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.79070001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94862024-02-07T18:58:41.567Z{"msg":"9cf049db-b96c-4db0-8857-909ea811979a is now active with session: 3a98678b-cc1b-4b5a-bb82-a351cb5658fa","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790713338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94872024-02-07T18:58:41.567Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790725526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479,"session_id":"3a98678b-cc1b-4b5a-bb82-a351cb5658fa"}
94882024-02-07T18:58:41.567Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:40.790744974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479}
94892024-02-07T18:58:41.567ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
94902024-02-07T18:58:41.567ZLoading write count information from file "/tmp/test_repair_verify.data"
94912024-02-07T18:58:41.567ZRun Repair workload
94922024-02-07T18:58:41.567Z01/30 Write block 331 len 40960 data: 7 7 8 8 6 7 6 8 10 8
94932024-02-07T18:58:41.567Z02/30 Read block 214 len 8192
94942024-02-07T18:58:41.567Z03/30 Read block 347 len 20480
94952024-02-07T18:58:41.567Z04/30 Read block 270 len 28672
94962024-02-07T18:58:41.567Z05/30 Write block 555 len 8192 data: 7 6
94972024-02-07T18:58:41.568Z06/30 Write block 189 len 24576 data: 4 4 8 9 8 8
94982024-02-07T18:58:41.568Z07/30 Read block 256 len 24576
94992024-02-07T18:58:41.568Z08/30 Write block 333 len 20480 data: 9 9 7 8 7
95002024-02-07T18:58:41.568Z09/30 Write block 584 len 40960 data: 5 6 7 6 7 7 7 8 8 7
95012024-02-07T18:58:41.568Z10/30 Read block 563 len 36864
95022024-02-07T18:58:41.572Z11/30 Read block 126 len 36864
95032024-02-07T18:58:41.572Z12/30 Read block 222 len 32768
95042024-02-07T18:58:41.572Z13/30 Write block 375 len 32768 data: 4 4 4 5 4 5 6 6
95052024-02-07T18:58:41.572Z14/30 Read block 522 len 12288
95062024-02-07T18:58:41.572Z15/30 Read block 489 len 28672
95072024-02-07T18:58:41.573Z16/30 Read block 491 len 16384
95082024-02-07T18:58:41.573Z17/30 Read block 46 len 16384
95092024-02-07T18:58:41.573Z18/30 Write block 50 len 8192 data: 6 9
95102024-02-07T18:58:41.573Z19/30 Flush
95112024-02-07T18:58:41.577Z20/30 Write block 86 len 36864 data: 5 4 5 5 7 7 8 8 10
95122024-02-07T18:58:41.577Z21/30 Write block 8 len 20480 data: 9 10 8 8 9
95132024-02-07T18:58:41.577Z22/30 Flush
95142024-02-07T18:58:41.582Z23/30 Write block 277 len 32768 data: 4 5 5 6 5 6 5 7
95152024-02-07T18:58:41.582Z24/30 Write block 370 len 40960 data: 7 6 5 5 4 5 5 5 6 5
95162024-02-07T18:58:41.582Z25/30 Read block 388 len 36864
95172024-02-07T18:58:41.590Z26/30 Read block 149 len 40960
95182024-02-07T18:58:41.590Z27/30 Write block 560 len 4096 data: 7
95192024-02-07T18:58:41.590Z28/30 Write block 383 len 28672 data: 6 5 5 7 7 8 9
95202024-02-07T18:58:41.590Z29/30 Write block 208 len 28672 data: 9 7 6 7 7 8 6
95212024-02-07T18:58:41.591Z30/30 Write block 367 len 20480 data: 7 8 7 8 7
95222024-02-07T18:58:41.591Z----------------------------------------------------------------
95232024-02-07T18:58:41.591Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
95242024-02-07T18:58:41.591ZStates: Active Active Active
95252024-02-07T18:58:41.591ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95262024-02-07T18:58:41.591Z 1 Acked 1000 Write 10 Done Sent Done false
95272024-02-07T18:58:41.591Z 2 Acked 1001 Read 2 Done Sent Done false
95282024-02-07T18:58:41.592Z 3 Acked 1002 Read 5 Done Sent Done false
95292024-02-07T18:58:41.592Z 4 Acked 1003 Read 7 Done Sent Done false
95302024-02-07T18:58:41.592Z 5 Acked 1004 Write 2 Done Sent Done false
95312024-02-07T18:58:41.592Z 6 Acked 1005 Write 6 Done Sent Done false
95322024-02-07T18:58:41.592Z 7 Acked 1006 Read 6 Done Sent Done false
95332024-02-07T18:58:41.592Z 8 Acked 1007 Write 5 Done Sent Done false
95342024-02-07T18:58:41.593Z 9 Acked 1008 Write 10 Done Sent Done false
95352024-02-07T18:58:41.593Z 10 Acked 1009 Read 9 Done Sent Done false
95362024-02-07T18:58:41.593Z 11 Acked 1010 Read 9 Done Sent Done false
95372024-02-07T18:58:41.593Z 12 Acked 1011 Read 8 Done Sent Done false
95382024-02-07T18:58:41.593Z 13 Acked 1012 Write 8 Done Sent Done false
95392024-02-07T18:58:41.593Z 14 Acked 1013 Read 3 Done Sent Done false
95402024-02-07T18:58:41.593Z 15 Acked 1014 Read 7 Done Sent Done false
95412024-02-07T18:58:41.593Z 16 Acked 1015 Read 4 Done Sent Done false
95422024-02-07T18:58:41.594Z 17 Acked 1016 Read 4 Done Sent Done false
95432024-02-07T18:58:41.594Z 18 Acked 1017 Write 2 Done Sent Done false
95442024-02-07T18:58:41.594Z 19 Acked 1018 Flush 0 Done Sent Done false
95452024-02-07T18:58:41.594Z 20 Acked 1019 Write 9 Done Sent Done false
95462024-02-07T18:58:41.595Z 21 Acked 1020 Write 5 Done Sent Done false
95472024-02-07T18:58:41.595Z 22 Acked 1021 Flush 0 Done Sent Done false
95482024-02-07T18:58:41.595Z 23 Acked 1022 Write 8 Done Sent Done false
95492024-02-07T18:58:41.595Z 24 Acked 1023 Write 10 Done Sent Done false
95502024-02-07T18:58:41.595Z 25 Acked 1024 Read 9 Done Sent Done false
95512024-02-07T18:58:41.595Z 26 Acked 1025 Read 10 Done Sent Done false
95522024-02-07T18:58:41.595Z 27 Acked 1026 Write 1 Done Sent Done false
95532024-02-07T18:58:41.596Z 28 Acked 1027 Write 7 Done Sent Done false
95542024-02-07T18:58:41.596Z 29 Acked 1028 Write 7 Sent Sent Sent false
95552024-02-07T18:58:41.596Z 30 Acked 1029 Write 5 Sent Sent Sent false
95562024-02-07T18:58:41.596Z STATES DS:0 DS:1 DS:2 TOTAL
95572024-02-07T18:58:41.596Z New 0 0 0 0
95582024-02-07T18:58:41.596Z Sent 2 30 2 34
95592024-02-07T18:58:41.597Z Done 28 0 28 56
95602024-02-07T18:58:41.597Z Skipped 0 0 0 0
95612024-02-07T18:58:41.597Z Error 0 0 0 0
95622024-02-07T18:58:41.597ZLast Flush: 1021 0 1021
95632024-02-07T18:58:41.597ZDownstairs last five completed:
95642024-02-07T18:58:41.597ZUpstairs last five completed: 30 29 28 27 26
95652024-02-07T18:58:41.598Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:40.822634641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1479}
95662024-02-07T18:58:41.598ZWrote out file "/tmp/test_repair_verify.data"
95672024-02-07T18:58:41.615Z
95682024-02-07T18:58:41.615ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
95692024-02-07T18:58:41.638ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95702024-02-07T18:58:41.638Z 0 000-019 86 81 86 141 134 141 F F F <---
95712024-02-07T18:58:41.639Z 1 020-039 85 85 85 139 139 139 F F F
95722024-02-07T18:58:41.639Z 2 040-059 86 77 86 140 123 140 F F F <---
95732024-02-07T18:58:41.639Z 3 060-079 80 80 80 131 131 131 F F F
95742024-02-07T18:58:41.639Z 4 080-099 86 84 86 141 137 141 F F F <---
95752024-02-07T18:58:41.639Z 5 100-119 85 85 85 139 139 139 F F F
95762024-02-07T18:58:41.640Z 6 120-139 84 84 84 138 138 138 F F F
95772024-02-07T18:58:41.640Z 7 140-159 84 84 84 138 138 138 F F F
95782024-02-07T18:58:41.640Z 8 160-179 82 82 82 135 135 135 F F F
95792024-02-07T18:58:41.640Z 9 180-199 86 75 86 140 121 140 F F F <---
95802024-02-07T18:58:41.640Z 10 200-219 80 80 80 131 131 131 T F T <---
95812024-02-07T18:58:41.641Z 11 220-239 85 85 85 139 139 139 F F F
95822024-02-07T18:58:41.641Z 12 240-259 83 83 83 136 136 136 F F F
95832024-02-07T18:58:41.641Z 13 260-279 84 84 84 137 137 137 T F T <---
95842024-02-07T18:58:41.641Z 14 280-299 80 80 80 131 131 131 T F T <---
95852024-02-07T18:58:41.641Z 15 300-319 80 80 80 129 129 129 F F F
95862024-02-07T18:58:41.642Z 16 320-339 86 81 86 140 134 140 F F F <---
95872024-02-07T18:58:41.642Z 17 340-359 86 78 86 140 127 140 F F F <---
95882024-02-07T18:58:41.642Z 18 360-379 86 82 86 140 135 140 T F T <---
95892024-02-07T18:58:41.642Z 19 380-399 86 85 86 140 139 140 T F T <---
95902024-02-07T18:58:41.642Z 20 400-419 82 82 82 135 135 135 F F F
95912024-02-07T18:58:41.642Z 21 420-439 85 85 85 139 139 139 F F F
95922024-02-07T18:58:41.642Z 22 440-459 84 84 84 138 138 138 F F F
95932024-02-07T18:58:41.643Z 23 460-479 83 83 83 136 136 136 F F F
95942024-02-07T18:58:41.643Z 24 480-499 84 84 84 138 138 138 F F F
95952024-02-07T18:58:41.643Z 25 500-519 84 84 84 138 138 138 F F F
95962024-02-07T18:58:41.643Z 26 520-539 80 80 80 130 130 130 F F F
95972024-02-07T18:58:41.643Z 27 540-559 86 77 86 140 123 140 F F F <---
95982024-02-07T18:58:41.644Z 28 560-579 85 85 85 139 139 139 T F T <---
95992024-02-07T18:58:41.644Z 29 580-599 86 84 86 140 138 140 F F F <---
96002024-02-07T18:58:41.644ZMax gen: 86, Max flush: 141
96012024-02-07T18:58:41.644ZError: Difference in extent metadata found!
96022024-02-07T18:58:41.644ZA Difference in extent metadata is expected here
96032024-02-07T18:58:41.645ZOn loop 43
96042024-02-07T18:58:41.645Z
96052024-02-07T18:58:41.645ZVerifying data now
96062024-02-07T18:58:46.306ZNo change after verify
96072024-02-07T18:58:46.306ZLoop: 43 Downstairs dump after verify (and repair):
96082024-02-07T18:58:46.319ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96092024-02-07T18:58:46.319Z 0 000-019 86 86 86 141 141 141 F F F
96102024-02-07T18:58:46.319Z 1 020-039 85 85 85 139 139 139 F F F
96112024-02-07T18:58:46.320Z 2 040-059 86 86 86 140 140 140 F F F
96122024-02-07T18:58:46.320Z 3 060-079 80 80 80 131 131 131 F F F
96132024-02-07T18:58:46.320Z 4 080-099 86 86 86 141 141 141 F F F
96142024-02-07T18:58:46.320Z 5 100-119 85 85 85 139 139 139 F F F
96152024-02-07T18:58:46.320Z 6 120-139 84 84 84 138 138 138 F F F
96162024-02-07T18:58:46.320Z 7 140-159 84 84 84 138 138 138 F F F
96172024-02-07T18:58:46.320Z 8 160-179 82 82 82 135 135 135 F F F
96182024-02-07T18:58:46.320Z 9 180-199 86 86 86 140 140 140 F F F
96192024-02-07T18:58:46.321Z 10 200-219 87 87 87 142 142 142 F F F
96202024-02-07T18:58:46.321Z 11 220-239 85 85 85 139 139 139 F F F
96212024-02-07T18:58:46.321Z 12 240-259 83 83 83 136 136 136 F F F
96222024-02-07T18:58:46.338Z 13 260-279 87 87 87 142 142 142 F F F
96232024-02-07T18:58:46.338Z 14 280-299 87 87 87 142 142 142 F F F
96242024-02-07T18:58:46.338Z 15 300-319 80 80 80 129 129 129 F F F
96252024-02-07T18:58:46.338Z 16 320-339 86 86 86 140 140 140 F F F
96262024-02-07T18:58:46.338Z 17 340-359 86 86 86 140 140 140 F F F
96272024-02-07T18:58:46.338Z 18 360-379 87 87 87 142 142 142 F F F
96282024-02-07T18:58:46.339Z 19 380-399 87 87 87 142 142 142 F F F
96292024-02-07T18:58:46.339Z 20 400-419 82 82 82 135 135 135 F F F
96302024-02-07T18:58:46.339Z 21 420-439 85 85 85 139 139 139 F F F
96312024-02-07T18:58:46.339Z 22 440-459 84 84 84 138 138 138 F F F
96322024-02-07T18:58:46.339Z 23 460-479 83 83 83 136 136 136 F F F
96332024-02-07T18:58:46.339Z 24 480-499 84 84 84 138 138 138 F F F
96342024-02-07T18:58:46.339Z 25 500-519 84 84 84 138 138 138 F F F
96352024-02-07T18:58:46.340Z 26 520-539 80 80 80 130 130 130 F F F
96362024-02-07T18:58:46.340Z 27 540-559 86 86 86 140 140 140 F F F
96372024-02-07T18:58:46.340Z 28 560-579 87 87 87 142 142 142 F F F
96382024-02-07T18:58:46.340Z 29 580-599 86 86 86 140 140 140 F F F
96392024-02-07T18:58:46.340ZMax gen: 87, Max flush: 142
96402024-02-07T18:58:46.340Z
96412024-02-07T18:58:46.340ZBegin loop 44
96422024-02-07T18:58:46.341ZDownstairs to restart: 1
96432024-02-07T18:58:46.366Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576375874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487}
96442024-02-07T18:58:46.366Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v"Crucible runtime is spawned
96452024-02-07T18:58:46.366Z:0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576726047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487}
96462024-02-07T18:58:46.366Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.57677295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487}
96472024-02-07T18:58:46.366Z{"msg":"Crucible a7017f06-a404-40de-99f5-7c66d6383dd8 has session id: 53be5707-eda3-42b6-a292-d388bcc20f7e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576798457Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96482024-02-07T18:58:46.366Z{"msg":"Upstairs opts: Upstairs UUID: a7017f06-a404-40de-99f5-7c66d6383dd8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.57684658Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96492024-02-07T18:58:46.367Z{"msg":"Crucible stats registered with UUID: a7017f06-a404-40de-99f5-7c66d6383dd8","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576882146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96502024-02-07T18:58:46.367Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576905992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487}
96512024-02-07T18:58:46.367Z{"msg":"a7017f06-a404-40de-99f5-7c66d6383dd8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576967104Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96522024-02-07T18:58:46.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.576994051Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96532024-02-07T18:58:46.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577011158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96542024-02-07T18:58:46.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577063461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96552024-02-07T18:58:46.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577111724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96562024-02-07T18:58:46.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.57714373Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96572024-02-07T18:58:46.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577172376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96582024-02-07T18:58:46.368Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577197563Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96592024-02-07T18:58:46.369Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577240827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96602024-02-07T18:58:46.369Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577284331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96612024-02-07T18:58:46.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577331855Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96622024-02-07T18:58:46.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577445609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96632024-02-07T18:58:46.369Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:45.577496212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96642024-02-07T18:58:46.369Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:45.577540217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96652024-02-07T18:58:46.370Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:45.577676458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96662024-02-07T18:58:46.370Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577704434Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96672024-02-07T18:58:46.370Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577719572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96682024-02-07T18:58:46.370Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.577830227Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96692024-02-07T18:58:46.370Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.578170311Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96702024-02-07T18:58:46.370Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.578206976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96712024-02-07T18:58:46.371Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.578222424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96722024-02-07T18:58:46.371Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.578236942Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96732024-02-07T18:58:46.371Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.578722817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96742024-02-07T18:58:46.398Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.57884425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96752024-02-07T18:58:46.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.57929306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96762024-02-07T18:58:46.398Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.579338194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96772024-02-07T18:58:46.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.57936217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96782024-02-07T18:58:46.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.579384797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96792024-02-07T18:58:46.399Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.579399095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96802024-02-07T18:58:46.399Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:45.579419563Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96812024-02-07T18:58:47.344Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.579162697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96822024-02-07T18:58:47.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.579238497Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96832024-02-07T18:58:47.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.580064935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96842024-02-07T18:58:47.355Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.58010128Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96852024-02-07T18:58:47.355Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.580648186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96862024-02-07T18:58:47.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581250765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96872024-02-07T18:58:47.356Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581295168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96882024-02-07T18:58:47.356Z{"msg":"[0]R flush_numbers[0..12]: [141, 139, 140, 131, 141, 139, 138, 138, 135, 140, 142, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581314646Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96892024-02-07T18:58:47.356Z{"msg":"[0]R generation[0..12]: [86, 85, 86, 80, 86, 85, 84, 84, 82, 86, 87, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581330734Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96902024-02-07T18:58:47.356Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581354471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96912024-02-07T18:58:47.357Z{"msg":"[1]R flush_numbers[0..12]: [141, 139, 140, 131, 141, 139, 138, 138, 135, 140, 142, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581369598Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96922024-02-07T18:58:47.357Z{"msg":"[1]R generation[0..12]: [86, 85, 86, 80, 86, 85, 84, 84, 82, 86, 87, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581390496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96932024-02-07T18:58:47.357Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581405633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96942024-02-07T18:58:47.357Z{"msg":"[2]R flush_numbers[0..12]: [141, 139, 140, 131, 141, 139, 138, 138, 135, 140, 142, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581426271Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96952024-02-07T18:58:47.357Z{"msg":"[2]R generation[0..12]: [86, 85, 86, 80, 86, 85, 84, 84, 82, 86, 87, 85]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581440529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96962024-02-07T18:58:47.357Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581461146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96972024-02-07T18:58:47.357Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581475314Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96982024-02-07T18:58:47.357Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581489432Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
96992024-02-07T18:58:47.357Z{"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581514379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97002024-02-07T18:58:47.357Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581528577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97012024-02-07T18:58:47.357ZWait for a query_work_queue command to finish before sending IO
97022024-02-07T18:58:47.357Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581555293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97032024-02-07T18:58:47.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581569671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97042024-02-07T18:58:47.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581590248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97052024-02-07T18:58:47.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581604607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97062024-02-07T18:58:47.358Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581619715Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97072024-02-07T18:58:47.358Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581633053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97082024-02-07T18:58:47.358Z{"msg":"a7017f06-a404-40de-99f5-7c66d6383dd8 is now active with session: 53be5707-eda3-42b6-a292-d388bcc20f7e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.5816528Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97092024-02-07T18:58:47.358Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581665588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487,"session_id":"53be5707-eda3-42b6-a292-d388bcc20f7e"}
97102024-02-07T18:58:47.358Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:46.581679056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487}
97112024-02-07T18:58:47.358ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97122024-02-07T18:58:47.358ZLoading write count information from file "/tmp/test_repair_verify.data"
97132024-02-07T18:58:47.359ZRun Repair workload
97142024-02-07T18:58:47.359Z01/30 Write block 8 len 8192 data: 10 11
97152024-02-07T18:58:47.359Z02/30 Read block 588 len 16384
97162024-02-07T18:58:47.359Z03/30 Write block 248 len 32768 data: 7 5 4 4 6 5 5 5
97172024-02-07T18:58:47.359Z04/30 Flush
97182024-02-07T18:58:47.369Z05/30 Write block 369 len 12288 data: 8 9 8
97192024-02-07T18:58:47.369Z06/30 Read block 47 len 40960
97202024-02-07T18:58:47.369Z07/30 Read block 521 len 4096
97212024-02-07T18:58:47.369Z08/30 Write block 447 len 36864 data: 8 8 6 6 7 5 7 6 5
97222024-02-07T18:58:47.370Z09/30 Read block 400 len 20480
97232024-02-07T18:58:47.370Z10/30 Read block 534 len 32768
97242024-02-07T18:58:47.370Z11/30 Read block 432 len 36864
97252024-02-07T18:58:47.370Z12/30 Write block 380 len 4096 data: 6
97262024-02-07T18:58:47.370Z13/30 Flush
97272024-02-07T18:58:47.371Z14/30 Write block 457 len 20480 data: 6 8 7 6 6
97282024-02-07T18:58:47.371Z15/30 Write block 296 len 28672 data: 7 9 8 5 5 5 6
97292024-02-07T18:58:47.371Z16/30 Read block 383 len 32768
97302024-02-07T18:58:47.387Z17/30 Write block 32 len 24576 data: 9 9 11 10 10 7
97312024-02-07T18:58:47.388Z18/30 Flush
97322024-02-07T18:58:47.388Z19/30 Write block 219 len 36864 data: 5 5 5 5 6 7 5 3 3
97332024-02-07T18:58:47.388Z20/30 Read block 321 len 32768
97342024-02-07T18:58:47.388Z21/30 Write block 476 len 4096 data: 7
97352024-02-07T18:58:47.388Z22/30 Read block 524 len 24576
97362024-02-07T18:58:47.389Z23/30 Write block 173 len 32768 data: 10 8 7 6 6 6 5 7
97372024-02-07T18:58:47.389Z24/30 Read block 255 len 32768
97382024-02-07T18:58:47.389Z25/30 Write block 469 len 20480 data: 3 5 5 4 4
97392024-02-07T18:58:47.389Z26/30 Read block 476 len 12288
97402024-02-07T18:58:47.389Z27/30 Read block 178 len 28672
97412024-02-07T18:58:47.389Z28/30 Write block 407 len 16384 data: 6 6 8 11
97422024-02-07T18:58:47.389Z29/30 Write block 559 len 28672 data: 7 8 6 5 5 8 7
97432024-02-07T18:58:47.390Z30/30 Read block 299 len 16384
97442024-02-07T18:58:47.390Z----------------------------------------------------------------
97452024-02-07T18:58:47.390Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:17
97462024-02-07T18:58:47.390ZStates: Active Active Active
97472024-02-07T18:58:47.390ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97482024-02-07T18:58:47.390Z 14 Acked 1013 Write 5 Done Done Done false
97492024-02-07T18:58:47.390Z 15 Acked 1014 Write 7 Done Done Done false
97502024-02-07T18:58:47.390Z 16 Acked 1015 Read 8 Done Sent Done false
97512024-02-07T18:58:47.391Z 17 Acked 1016 Write 6 Done Sent Done false
97522024-02-07T18:58:47.391Z 18 Acked 1017 Flush 0 Done Sent Done false
97532024-02-07T18:58:47.391Z 19 Acked 1018 Write 9 Done Sent Done false
97542024-02-07T18:58:47.391Z 20 Acked 1019 Read 8 Done Sent Done false
97552024-02-07T18:58:47.392Z 21 Acked 1020 Write 1 Done Sent Done false
97562024-02-07T18:58:47.392Z 22 Acked 1021 Read 6 Done Sent Done false
97572024-02-07T18:58:47.392Z 23 Acked 1022 Write 8 Done Sent Done false
97582024-02-07T18:58:47.392Z 24 Acked 1023 Read 8 Done Sent Done false
97592024-02-07T18:58:47.393Z 25 Acked 1024 Write 5 Done Sent Done false
97602024-02-07T18:58:47.393Z 26 Acked 1025 Read 3 Done Sent Done false
97612024-02-07T18:58:47.393Z 27 Acked 1026 Read 7 Done Sent Done false
97622024-02-07T18:58:47.393Z 28 Acked 1027 Write 4 Done Sent Done false
97632024-02-07T18:58:47.393Z 29 Acked 1028 Write 7 Done Sent Done false
97642024-02-07T18:58:47.393Z 30 Acked 1029 Read 4 Done Sent Sent false
97652024-02-07T18:58:47.393Z STATES DS:0 DS:1 DS:2 TOTAL
97662024-02-07T18:58:47.393Z New 0 0 0 0
97672024-02-07T18:58:47.393Z Sent 0 15 1 16
97682024-02-07T18:58:47.394Z Done 17 2 16 35
97692024-02-07T18:58:47.394Z Skipped 0 0 0 0
97702024-02-07T18:58:47.394Z Error 0 0 0 0
97712024-02-07T18:58:47.394ZLast Flush: 1017 1012 1017
97722024-02-07T18:58:47.394ZDownstairs last five completed: 1012 1011 1010 1009 1008
97732024-02-07T18:58:47.395ZUpstairs last five completed: 30 29 28 27 26
97742024-02-07T18:58:47.395Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:46.615916476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1487}
97752024-02-07T18:58:47.411ZWrote out file "/tmp/test_repair_verify.data"
97762024-02-07T18:58:47.412Z
97772024-02-07T18:58:47.412ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97782024-02-07T18:58:47.428ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97792024-02-07T18:58:47.428Z 0 000-019 88 88 88 143 143 143 F F F
97802024-02-07T18:58:47.428Z 1 020-039 88 85 88 145 139 145 F F F <---
97812024-02-07T18:58:47.428Z 2 040-059 86 86 86 140 140 140 F F F
97822024-02-07T18:58:47.429Z 3 060-079 80 80 80 131 131 131 F F F
97832024-02-07T18:58:47.429Z 4 080-099 86 86 86 141 141 141 F F F
97842024-02-07T18:58:47.429Z 5 100-119 85 85 85 139 139 139 F F F
97852024-02-07T18:58:47.429Z 6 120-139 84 84 84 138 138 138 F F F
97862024-02-07T18:58:47.430Z 7 140-159 84 84 84 138 138 138 F F F
97872024-02-07T18:58:47.430Z 8 160-179 82 82 82 135 135 135 T F T <---
97882024-02-07T18:58:47.430Z 9 180-199 86 86 86 140 140 140 T F T <---
97892024-02-07T18:58:47.430Z 10 200-219 87 87 87 142 142 142 T F T <---
97902024-02-07T18:58:47.430Z 11 220-239 85 85 85 139 139 139 T F T <---
97912024-02-07T18:58:47.430Z 12 240-259 88 88 88 143 143 143 F F F
97922024-02-07T18:58:47.430Z 13 260-279 87 87 87 142 142 142 F F F
97932024-02-07T18:58:47.431Z 14 280-299 88 87 88 145 142 145 F T F <---
97942024-02-07T18:58:47.431Z 15 300-319 88 80 88 145 129 145 F T F <---
97952024-02-07T18:58:47.431Z 16 320-339 86 86 86 140 140 140 F F F
97962024-02-07T18:58:47.431Z 17 340-359 86 86 86 140 140 140 F F F
97972024-02-07T18:58:47.431Z 18 360-379 88 88 88 144 144 144 F F F
97982024-02-07T18:58:47.432Z 19 380-399 88 88 88 144 144 144 F F F
97992024-02-07T18:58:47.432Z 20 400-419 82 82 82 135 135 135 T F T <---
98002024-02-07T18:58:47.432Z 21 420-439 85 85 85 139 139 139 F F F
98012024-02-07T18:58:47.432Z 22 440-459 88 88 88 145 144 145 F T F <---
98022024-02-07T18:58:47.433Z 23 460-479 88 83 88 145 136 145 T T T <---
98032024-02-07T18:58:47.433Z 24 480-499 84 84 84 138 138 138 F F F
98042024-02-07T18:58:47.433Z 25 500-519 84 84 84 138 138 138 F F F
98052024-02-07T18:58:47.433Z 26 520-539 80 80 80 130 130 130 F F F
98062024-02-07T18:58:47.433Z 27 540-559 86 86 86 140 140 140 T F T <---
98072024-02-07T18:58:47.434Z 28 560-579 87 87 87 142 142 142 T F T <---
98082024-02-07T18:58:47.434Z 29 580-599 86 86 86 140 140 140 F F F
98092024-02-07T18:58:47.434ZMax gen: 88, Max flush: 145
98102024-02-07T18:58:47.434ZError: Difference in extent metadata found!
98112024-02-07T18:58:47.435ZA Difference in extent metadata is expected here
98122024-02-07T18:58:47.435ZOn loop 44
98132024-02-07T18:58:47.435Z
98142024-02-07T18:58:47.435ZVerifying data now
98152024-02-07T18:58:53.022ZNo change after verify
98162024-02-07T18:58:53.022ZLoop: 44 Downstairs dump after verify (and repair):
98172024-02-07T18:58:53.043ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98182024-02-07T18:58:53.043Z 0 000-019 88 88 88 143 143 143 F F F
98192024-02-07T18:58:53.043Z 1 020-039 88 88 88 145 145 145 F F F
98202024-02-07T18:58:53.043Z 2 040-059 86 86 86 140 140 140 F F F
98212024-02-07T18:58:53.043Z 3 060-079 80 80 80 131 131 131 F F F
98222024-02-07T18:58:53.043Z 4 080-099 86 86 86 141 141 141 F F F
98232024-02-07T18:58:53.043Z 5 100-119 85 85 85 139 139 139 F F F
98242024-02-07T18:58:53.043Z 6 120-139 84 84 84 138 138 138 F F F
98252024-02-07T18:58:53.043Z 7 140-159 84 84 84 138 138 138 F F F
98262024-02-07T18:58:53.044Z 8 160-179 89 89 89 146 146 146 F F F
98272024-02-07T18:58:53.044Z 9 180-199 89 89 89 146 146 146 F F F
98282024-02-07T18:58:53.044Z 10 200-219 89 89 89 146 146 146 F F F
98292024-02-07T18:58:53.044Z 11 220-239 89 89 89 146 146 146 F F F
98302024-02-07T18:58:53.044Z 12 240-259 88 88 88 143 143 143 F F F
98312024-02-07T18:58:53.044Z 13 260-279 87 87 87 142 142 142 F F F
98322024-02-07T18:58:53.044Z 14 280-299 88 88 88 145 145 145 F F F
98332024-02-07T18:58:53.044Z 15 300-319 88 88 88 145 145 145 F F F
98342024-02-07T18:58:53.044Z 16 320-339 86 86 86 140 140 140 F F F
98352024-02-07T18:58:53.044Z 17 340-359 86 86 86 140 140 140 F F F
98362024-02-07T18:58:53.045Z 18 360-379 88 88 88 144 144 144 F F F
98372024-02-07T18:58:53.045Z 19 380-399 88 88 88 144 144 144 F F F
98382024-02-07T18:58:53.045Z 20 400-419 89 89 89 146 146 146 F F F
98392024-02-07T18:58:53.045Z 21 420-439 85 85 85 139 139 139 F F F
98402024-02-07T18:58:53.045Z 22 440-459 88 88 88 145 145 145 F F F
98412024-02-07T18:58:53.045Z 23 460-479 89 89 89 146 146 146 F F F
98422024-02-07T18:58:53.045Z 24 480-499 84 84 84 138 138 138 F F F
98432024-02-07T18:58:53.046Z 25 500-519 84 84 84 138 138 138 F F F
98442024-02-07T18:58:53.046Z 26 520-539 80 80 80 130 130 130 F F F
98452024-02-07T18:58:53.046Z 27 540-559 89 89 89 146 146 146 F F F
98462024-02-07T18:58:53.046Z 28 560-579 89 89 89 146 146 146 F F F
98472024-02-07T18:58:53.046Z 29 580-599 86 86 86 140 140 140 F F F
98482024-02-07T18:58:53.046ZMax gen: 89, Max flush: 146
98492024-02-07T18:58:53.061Z
98502024-02-07T18:58:53.062ZBegin loop 45
98512024-02-07T18:58:53.062ZDownstairs to restart: 0
98522024-02-07T18:58:53.063Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287204432Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495}
98532024-02-07T18:58:53.063Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\nCrucible runtime is spawned
98542024-02-07T18:58:53.063Z rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287558214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495}
98552024-02-07T18:58:53.063Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287612197Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495}
98562024-02-07T18:58:53.063Z{"msg":"Crucible cd79421b-83ce-4ec7-9ee1-0272c4029522 has session id: 49982a86-686e-4ac4-ad41-5535c18b4397","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287643372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98572024-02-07T18:58:53.063Z{"msg":"Upstairs opts: Upstairs UUID: cd79421b-83ce-4ec7-9ee1-0272c4029522, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287692116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98582024-02-07T18:58:53.064Z{"msg":"Crucible stats registered with UUID: cd79421b-83ce-4ec7-9ee1-0272c4029522","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287711873Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98592024-02-07T18:58:53.064Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287737189Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495}
98602024-02-07T18:58:53.064Z{"msg":"cd79421b-83ce-4ec7-9ee1-0272c4029522 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287752578Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98612024-02-07T18:58:53.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287783863Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98622024-02-07T18:58:53.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287802341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98632024-02-07T18:58:53.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287820278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98642024-02-07T18:58:53.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287862213Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98652024-02-07T18:58:53.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287893508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98662024-02-07T18:58:53.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287912886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98672024-02-07T18:58:53.064Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.28795495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98682024-02-07T18:58:53.065Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.287983866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98692024-02-07T18:58:53.065Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288008593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98702024-02-07T18:58:53.065Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:52.288152573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98712024-02-07T18:58:53.065Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:52.288204156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98722024-02-07T18:58:53.065Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288244571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98732024-02-07T18:58:53.065Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288275867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98742024-02-07T18:58:53.066Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:52.288308992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98752024-02-07T18:58:53.066Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288343338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98762024-02-07T18:58:53.066Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288381213Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98772024-02-07T18:58:53.066Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288426156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98782024-02-07T18:58:53.071Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288847749Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98792024-02-07T18:58:53.072Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288888994Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98802024-02-07T18:58:53.072Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288904612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98812024-02-07T18:58:53.072Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.288921419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98822024-02-07T18:58:53.072Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.289539116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98832024-02-07T18:58:53.072Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.289574801Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98842024-02-07T18:58:53.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.290163651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98852024-02-07T18:58:53.072Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.290208355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98862024-02-07T18:58:53.072Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.290227503Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98872024-02-07T18:58:53.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.290244101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98882024-02-07T18:58:53.072Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.290271597Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98892024-02-07T18:58:53.072Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:52.290289404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98902024-02-07T18:58:54.057Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.288711157Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98912024-02-07T18:58:54.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.288820512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98922024-02-07T18:58:54.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.289695644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98932024-02-07T18:58:54.496Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.289741758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98942024-02-07T18:58:54.496Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.29024629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98952024-02-07T18:58:54.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.290840669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98962024-02-07T18:58:54.497Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.290890812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98972024-02-07T18:58:54.497Z{"msg":"[0]R flush_numbers[0..12]: [143, 145, 140, 131, 141, 139, 138, 138, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.290911669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98982024-02-07T18:58:54.497Z{"msg":"[0]R generation[0..12]: [88, 88, 86, 80, 86, 85, 84, 84, 89, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.290948205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
98992024-02-07T18:58:54.497Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.29098279Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99002024-02-07T18:58:54.498Z{"msg":"[1]R flush_numbers[0..12]: [143, 145, 140, 131, 141, 139, 138, 138, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291013566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99012024-02-07T18:58:54.498Z{"msg":"[1]R generation[0..12]: [88, 88, 86, 80, 86, 85, 84, 84, 89, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291031953Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99022024-02-07T18:58:54.498Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO
99032024-02-07T18:58:54.498Z:"2024-02-07T18:58:53.291049761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99042024-02-07T18:58:54.498Z{"msg":"[2]R flush_numbers[0..12]: [143, 145, 140, 131, 141, 139, 138, 138, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291082257Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99052024-02-07T18:58:54.499Z{"msg":"[2]R generation[0..12]: [88, 88, 86, 80, 86, 85, 84, 84, 89, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291101864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99062024-02-07T18:58:54.499Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.2911303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99072024-02-07T18:58:54.499Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291147868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99082024-02-07T18:58:54.499Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291181283Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99092024-02-07T18:58:54.499Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291199761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99102024-02-07T18:58:54.500Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291238115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99112024-02-07T18:58:54.500Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.29127468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99122024-02-07T18:58:54.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291300397Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99132024-02-07T18:58:54.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291328223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99142024-02-07T18:58:54.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291346101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99152024-02-07T18:58:54.501Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291372257Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99162024-02-07T18:58:54.501Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291389755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99172024-02-07T18:58:54.501Z{"msg":"cd79421b-83ce-4ec7-9ee1-0272c4029522 is now active with session: 49982a86-686e-4ac4-ad41-5535c18b4397","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291420241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99182024-02-07T18:58:54.501Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291437218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495,"session_id":"49982a86-686e-4ac4-ad41-5535c18b4397"}
99192024-02-07T18:58:54.501Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:53.291478243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495}
99202024-02-07T18:58:54.501ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99212024-02-07T18:58:54.502ZLoading write count information from file "/tmp/test_repair_verify.data"
99222024-02-07T18:58:54.502ZRun Repair workload
99232024-02-07T18:58:54.502Z01/30 Write block 64 len 12288 data: 7 7 10
99242024-02-07T18:58:54.502Z02/30 Write block 64 len 28672 data: 8 8 11 10 10 10 9
99252024-02-07T18:58:54.502Z03/30 Read block 476 len 40960
99262024-02-07T18:58:54.503Z04/30 Write block 63 len 4096 data: 5
99272024-02-07T18:58:54.504Z05/30 Read block 94 len 12288
99282024-02-07T18:58:54.504Z06/30 Read block 509 len 16384
99292024-02-07T18:58:54.504Z07/30 Write block 105 len 16384 data: 10 9 10 10
99302024-02-07T18:58:54.504Z08/30 Read block 136 len 8192
99312024-02-07T18:58:54.505Z09/30 Read block 47 len 40960
99322024-02-07T18:58:54.505Z10/30 Write block 562 len 36864 data: 6 6 9 8 7 8 7 7 8
99332024-02-07T18:58:54.505Z11/30 Read block 326 len 16384
99342024-02-07T18:58:54.505Z12/30 Flush
99352024-02-07T18:58:54.505Z13/30 Write block 154 len 20480 data: 9 5 4 3 6
99362024-02-07T18:58:54.506Z14/30 Write block 284 len 36864 data: 8 8 10 11 11 11 12 10 8
99372024-02-07T18:58:54.506Z15/30 Read block 9 len 36864
99382024-02-07T18:58:54.506Z16/30 Write block 328 len 20480 data: 5 5 6 8 8
99392024-02-07T18:58:54.506Z17/30 Flush
99402024-02-07T18:58:54.506Z18/30 Write block 547 len 20480 data: 7 8 9 8 7
99412024-02-07T18:58:54.506Z19/30 Write block 505 len 12288 data: 11 8 7
99422024-02-07T18:58:54.507Z20/30 Read block 508 len 16384
99432024-02-07T18:58:54.507Z21/30 Write block 568 len 8192 data: 8 8
99442024-02-07T18:58:54.507Z22/30 Write block 451 len 40960 data: 8 6 8 7 6 4 7 9 8 7
99452024-02-07T18:58:54.507Z23/30 Flush
99462024-02-07T18:58:54.507Z24/30 Write block 399 len 32768 data: 5 6 5 7 5 5 4 6
99472024-02-07T18:58:54.507Z25/30 Read block 456 len 32768
99482024-02-07T18:58:54.508Z26/30 Read block 187 len 40960
99492024-02-07T18:58:54.508Z27/30 Write block 60 len 36864 data: 6 6 5 6 9 9 12 11 11
99502024-02-07T18:58:54.508Z28/30 Read block 304 len 24576
99512024-02-07T18:58:54.508Z29/30 Read block 230 len 40960
99522024-02-07T18:58:54.508Z30/30 Write block 424 len 24576 data: 5 10 9 8 11 10
99532024-02-07T18:58:54.509Z----------------------------------------------------------------
99542024-02-07T18:58:54.509Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
99552024-02-07T18:58:54.509ZStates: Active Active Active
99562024-02-07T18:58:54.509ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99572024-02-07T18:58:54.509Z 1 Acked 1000 Write 3 Done Done Done false
99582024-02-07T18:58:54.509Z 2 Acked 1001 Write 7 Done Done Done false
99592024-02-07T18:58:54.510Z 3 Acked 1002 Read 10 Sent Done Done false
99602024-02-07T18:58:54.510Z 4 Acked 1003 Write 1 Sent Done Done false
99612024-02-07T18:58:54.510Z 5 Acked 1004 Read 3 Sent Done Done false
99622024-02-07T18:58:54.510Z 6 Acked 1005 Read 4 Sent Done Done false
99632024-02-07T18:58:54.510Z 7 Acked 1006 Write 4 Sent Done Done false
99642024-02-07T18:58:54.511Z 8 Acked 1007 Read 2 Sent Done Done false
99652024-02-07T18:58:54.518Z 9 Acked 1008 Read 10 Sent Done Done false
99662024-02-07T18:58:54.519Z 10 Acked 1009 Write 9 Sent Done Done false
99672024-02-07T18:58:54.519Z 11 Acked 1010 Read 4 Sent Done Done false
99682024-02-07T18:58:54.519Z 12 Acked 1011 Flush 0 Sent Done Done false
99692024-02-07T18:58:54.519Z 13 Acked 1012 Write 5 Sent Done Done false
99702024-02-07T18:58:54.519Z 14 Acked 1013 Write 9 Sent Done Done false
99712024-02-07T18:58:54.520Z 15 Acked 1014 Read 9 Sent Done Done false
99722024-02-07T18:58:54.520Z 16 Acked 1015 Write 5 Sent Done Done false
99732024-02-07T18:58:54.520Z 17 Acked 1016 Flush 0 Sent Done Done false
99742024-02-07T18:58:54.521Z 18 Acked 1017 Write 5 Sent Done Done false
99752024-02-07T18:58:54.521Z 19 Acked 1018 Write 3 Sent Done Done false
99762024-02-07T18:58:54.521Z 20 Acked 1019 Read 4 Sent Done Done false
99772024-02-07T18:58:54.521Z 21 Acked 1020 Write 2 Sent Done Done false
99782024-02-07T18:58:54.521Z 22 Acked 1021 Write 10 Sent Done Done false
99792024-02-07T18:58:54.522Z 23 Acked 1022 Flush 0 Sent Done Done false
99802024-02-07T18:58:54.522Z 24 Acked 1023 Write 8 Sent Done Done false
99812024-02-07T18:58:54.522Z 25 Acked 1024 Read 8 Sent Done Done false
99822024-02-07T18:58:54.522Z 26 Acked 1025 Read 10 Sent Done Done false
99832024-02-07T18:58:54.523Z 27 Acked 1026 Write 9 Sent Done Done false
99842024-02-07T18:58:54.523Z 28 Acked 1027 Read 6 Sent Done Done false
99852024-02-07T18:58:54.523Z 29 Acked 1028 Read 10 Sent Done Done false
99862024-02-07T18:58:54.523Z 30 Acked 1029 Write 6 Sent Sent Sent false
99872024-02-07T18:58:54.523Z STATES DS:0 DS:1 DS:2 TOTAL
99882024-02-07T18:58:54.524Z New 0 0 0 0
99892024-02-07T18:58:54.524Z Sent 28 1 1 30
99902024-02-07T18:58:54.524Z Done 2 29 29 60
99912024-02-07T18:58:54.524Z Skipped 0 0 0 0
99922024-02-07T18:58:54.546Z Error 0 0 0 0
99932024-02-07T18:58:54.546ZLast Flush: 0 1022 1022
99942024-02-07T18:58:54.546ZDownstairs last five completed:
99952024-02-07T18:58:54.546ZUpstairs last five completed: 30 29 28 27 26
99962024-02-07T18:58:54.546Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:53.325330365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1495}
99972024-02-07T18:58:54.547ZWrote out file "/tmp/test_repair_verify.data"
99982024-02-07T18:58:54.547Z
99992024-02-07T18:58:54.547ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100002024-02-07T18:58:54.547ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100012024-02-07T18:58:54.547Z 0 000-019 88 88 88 143 143 143 F F F
100022024-02-07T18:58:54.548Z 1 020-039 88 88 88 145 145 145 F F F
100032024-02-07T18:58:54.548Z 2 040-059 86 86 86 140 140 140 F F F
100042024-02-07T18:58:54.548Z 3 060-079 80 90 90 131 147 147 T T T <---
100052024-02-07T18:58:54.548Z 4 080-099 86 86 86 141 141 141 F F F
100062024-02-07T18:58:54.548Z 5 100-119 85 90 90 139 147 147 F F F <---
100072024-02-07T18:58:54.549Z 6 120-139 84 84 84 138 138 138 F F F
100082024-02-07T18:58:54.549Z 7 140-159 84 90 90 138 148 148 F F F <---
100092024-02-07T18:58:54.549Z 8 160-179 89 89 89 146 146 146 F F F
100102024-02-07T18:58:54.549Z 9 180-199 89 89 89 146 146 146 F F F
100112024-02-07T18:58:54.549Z 10 200-219 89 89 89 146 146 146 F F F
100122024-02-07T18:58:54.549Z 11 220-239 89 89 89 146 146 146 F F F
100132024-02-07T18:58:54.550Z 12 240-259 88 88 88 143 143 143 F F F
100142024-02-07T18:58:54.550Z 13 260-279 87 87 87 142 142 142 F F F
100152024-02-07T18:58:54.550Z 14 280-299 88 90 90 145 148 148 F F F <---
100162024-02-07T18:58:54.550Z 15 300-319 88 88 88 145 145 145 F F F
100172024-02-07T18:58:54.550Z 16 320-339 86 90 90 140 148 148 F F F <---
100182024-02-07T18:58:54.550Z 17 340-359 86 86 86 140 140 140 F F F
100192024-02-07T18:58:54.551Z 18 360-379 88 88 88 144 144 144 F F F
100202024-02-07T18:58:54.551Z 19 380-399 88 88 88 144 144 144 F T T <---
100212024-02-07T18:58:54.551Z 20 400-419 89 89 89 146 146 146 F T T <---
100222024-02-07T18:58:54.551Z 21 420-439 85 85 85 139 139 139 F T T <---
100232024-02-07T18:58:54.551Z 22 440-459 88 90 90 145 149 149 F F F <---
100242024-02-07T18:58:54.551Z 23 460-479 89 90 90 146 149 149 F F F <---
100252024-02-07T18:58:54.552Z 24 480-499 84 84 84 138 138 138 F F F
100262024-02-07T18:58:54.552Z 25 500-519 84 90 90 138 149 149 F F F <---
100272024-02-07T18:58:54.552Z 26 520-539 80 80 80 130 130 130 F F F
100282024-02-07T18:58:54.552Z 27 540-559 89 90 90 146 149 149 F F F <---
100292024-02-07T18:58:54.553Z 28 560-579 89 90 90 146 149 149 F F F <---
100302024-02-07T18:58:54.553Z 29 580-599 86 86 86 140 140 140 F F F
100312024-02-07T18:58:54.553ZMax gen: 90, Max flush: 149
100322024-02-07T18:58:54.554ZError: Difference in extent metadata found!
100332024-02-07T18:58:54.554ZA Difference in extent metadata is expected here
100342024-02-07T18:58:54.554ZOn loop 45
100352024-02-07T18:58:54.554Z
100362024-02-07T18:58:54.554ZVerifying data now
100372024-02-07T18:58:59.766ZNo change after verify
100382024-02-07T18:58:59.766ZLoop: 45 Downstairs dump after verify (and repair):
100392024-02-07T18:58:59.781ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100402024-02-07T18:58:59.781Z 0 000-019 88 88 88 143 143 143 F F F
100412024-02-07T18:58:59.782Z 1 020-039 88 88 88 145 145 145 F F F
100422024-02-07T18:58:59.782Z 2 040-059 86 86 86 140 140 140 F F F
100432024-02-07T18:58:59.782Z 3 060-079 91 91 91 150 150 150 F F F
100442024-02-07T18:58:59.782Z 4 080-099 86 86 86 141 141 141 F F F
100452024-02-07T18:58:59.782Z 5 100-119 90 90 90 147 147 147 F F F
100462024-02-07T18:58:59.783Z 6 120-139 84 84 84 138 138 138 F F F
100472024-02-07T18:58:59.783Z 7 140-159 90 90 90 148 148 148 F F F
100482024-02-07T18:58:59.783Z 8 160-179 89 89 89 146 146 146 F F F
100492024-02-07T18:58:59.783Z 9 180-199 89 89 89 146 146 146 F F F
100502024-02-07T18:58:59.783Z 10 200-219 89 89 89 146 146 146 F F F
100512024-02-07T18:58:59.788Z 11 220-239 89 89 89 146 146 146 F F F
100522024-02-07T18:58:59.788Z 12 240-259 88 88 88 143 143 143 F F F
100532024-02-07T18:58:59.788Z 13 260-279 87 87 87 142 142 142 F F F
100542024-02-07T18:58:59.788Z 14 280-299 90 90 90 148 148 148 F F F
100552024-02-07T18:58:59.789Z 15 300-319 88 88 88 145 145 145 F F F
100562024-02-07T18:58:59.789Z 16 320-339 90 90 90 148 148 148 F F F
100572024-02-07T18:58:59.789Z 17 340-359 86 86 86 140 140 140 F F F
100582024-02-07T18:58:59.789Z 18 360-379 88 88 88 144 144 144 F F F
100592024-02-07T18:58:59.789Z 19 380-399 91 91 91 150 150 150 F F F
100602024-02-07T18:58:59.789Z 20 400-419 91 91 91 150 150 150 F F F
100612024-02-07T18:58:59.789Z 21 420-439 91 91 91 150 150 150 F F F
100622024-02-07T18:58:59.790Z 22 440-459 90 90 90 149 149 149 F F F
100632024-02-07T18:58:59.790Z 23 460-479 90 90 90 149 149 149 F F F
100642024-02-07T18:58:59.790Z 24 480-499 84 84 84 138 138 138 F F F
100652024-02-07T18:58:59.790Z 25 500-519 90 90 90 149 149 149 F F F
100662024-02-07T18:58:59.790Z 26 520-539 80 80 80 130 130 130 F F F
100672024-02-07T18:58:59.790Z 27 540-559 90 90 90 149 149 149 F F F
100682024-02-07T18:58:59.790Z 28 560-579 90 90 90 149 149 149 F F F
100692024-02-07T18:58:59.790Z 29 580-599 86 86 86 140 140 140 F F F
100702024-02-07T18:58:59.790ZMax gen: 91, Max flush: 150
100712024-02-07T18:58:59.791Z
100722024-02-07T18:58:59.791ZBegin loop 46
100732024-02-07T18:58:59.791ZDownstairs to restart: 2
100742024-02-07T18:58:59.803Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037316794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503}
100752024-02-07T18:58:59.804Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037574369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503}
100762024-02-07T18:58:59.817Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037610235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503}
100772024-02-07T18:58:59.818Z{"msg":"Crucible e7115a89-f52f-4873-9bf2-b2dd0e66a686 has session id: db8a66ed-950f-4e29-b4a7-cae23f0c0b76","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037635571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100782024-02-07T18:58:59.818Z{"msg":"Upstairs opts: Upstairs UUID: e7115a89-f52f-4873-9bf2-b2dd0e66a686, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037679865Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100792024-02-07T18:58:59.818Z{"msg":"Crucible runtime is spawned
100802024-02-07T18:58:59.818ZCrucible stats registered with UUID: e7115a89-f52f-4873-9bf2-b2dd0e66a686","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037728268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100812024-02-07T18:58:59.818Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037800119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503}
100822024-02-07T18:58:59.818Z{"msg":"e7115a89-f52f-4873-9bf2-b2dd0e66a686 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037827845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100832024-02-07T18:58:59.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037842913Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100842024-02-07T18:58:59.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037877128Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100852024-02-07T18:58:59.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037912683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100862024-02-07T18:58:59.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037952688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100872024-02-07T18:58:59.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.037990923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100882024-02-07T18:58:59.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038026298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100892024-02-07T18:58:59.820Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038050755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100902024-02-07T18:58:59.820Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038131034Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100912024-02-07T18:58:59.821Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038201704Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100922024-02-07T18:58:59.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038244959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100932024-02-07T18:58:59.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038347225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100942024-02-07T18:58:59.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038377551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100952024-02-07T18:58:59.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038847477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100962024-02-07T18:58:59.822Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038907149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100972024-02-07T18:58:59.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.038962921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100982024-02-07T18:58:59.822Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.039013925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
100992024-02-07T18:58:59.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.039460474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101002024-02-07T18:58:59.822Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.039595326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101012024-02-07T18:58:59.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.039895675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101022024-02-07T18:58:59.823Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.039926081Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101032024-02-07T18:58:59.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.040072211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101042024-02-07T18:58:59.823Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.040108217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101052024-02-07T18:58:59.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.040124284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101062024-02-07T18:58:59.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.04015866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101072024-02-07T18:58:59.823Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.040180647Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101082024-02-07T18:58:59.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.040195005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101092024-02-07T18:58:59.823Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.040558956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101102024-02-07T18:58:59.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041065207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101112024-02-07T18:58:59.823Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041110811Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101122024-02-07T18:58:59.823Z{"msg":"[0]R flush_numbers[0..12]: [143, 145, 140, 150, 141, 147, 138, 148, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041131508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101132024-02-07T18:58:59.824Z{"msg":"[0]R generation[0..12]: [88, 88, 86, 91, 86, 90, 84, 90, 89, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041157805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101142024-02-07T18:58:59.824Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041177072Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101152024-02-07T18:58:59.824Z{"msg":"[1]R flush_numbers[0..12]: [143, 145, 140, 150, 141, 147, 138, 148, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.04119508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101162024-02-07T18:58:59.824Z{"msg":"[1]R generation[0..12]: [88, 88, 86, 91, 86, 90, 84, 90, 89, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041227715Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101172024-02-07T18:58:59.824Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
101182024-02-07T18:58:59.825Z,"time":"2024-02-07T18:58:59.041246203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101192024-02-07T18:58:59.825Z{"msg":"[2]R flush_numbers[0..12]: [143, 145, 140, 150, 141, 147, 138, 148, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.04126599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101202024-02-07T18:58:59.825Z{"msg":"[2]R generation[0..12]: [88, 88, 86, 91, 86, 90, 84, 90, 89, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041283168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101212024-02-07T18:58:59.825Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041309014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101222024-02-07T18:58:59.825Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041326282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101232024-02-07T18:58:59.825Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.04134411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101242024-02-07T18:58:59.825Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041360417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101252024-02-07T18:58:59.825Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041384844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101262024-02-07T18:58:59.826Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041402112Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101272024-02-07T18:58:59.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041425868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101282024-02-07T18:58:59.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041455324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101292024-02-07T18:58:59.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.04148444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101302024-02-07T18:58:59.826Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041502048Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101312024-02-07T18:58:59.826Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041518276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101322024-02-07T18:58:59.826Z{"msg":"e7115a89-f52f-4873-9bf2-b2dd0e66a686 is now active with session: db8a66ed-950f-4e29-b4a7-cae23f0c0b76","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041540303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101332024-02-07T18:58:59.826Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041567009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503,"session_id":"db8a66ed-950f-4e29-b4a7-cae23f0c0b76"}
101342024-02-07T18:58:59.827Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:58:59.041584327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503}
101352024-02-07T18:58:59.827ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101362024-02-07T18:58:59.827ZLoading write count information from file "/tmp/test_repair_verify.data"
101372024-02-07T18:58:59.827ZRun Repair workload
101382024-02-07T18:58:59.827Z01/30 Write block 50 len 28672 data: 7 10 8 9 10 11 11
101392024-02-07T18:58:59.842Z02/30 Write block 436 len 40960 data: 8 10 6 8 9 9 10 10 10 10
101402024-02-07T18:58:59.842Z03/30 Read block 540 len 12288
101412024-02-07T18:58:59.842Z04/30 Flush
101422024-02-07T18:58:59.842Z05/30 Write block 354 len 4096 data: 7
101432024-02-07T18:58:59.842Z06/30 Write block 371 len 8192 data: 9 6
101442024-02-07T18:58:59.842Z07/30 Write block 331 len 16384 data: 9 9 10 10
101452024-02-07T18:58:59.842Z08/30 Write block 558 len 40960 data: 6 8 9 7 7 7 10 9 8 9
101462024-02-07T18:58:59.842Z09/30 Read block 150 len 36864
101472024-02-07T18:58:59.843Z10/30 Write block 200 len 16384 data: 5 4 7 5
101482024-02-07T18:58:59.843Z11/30 Write block 133 len 4096 data: 7
101492024-02-07T18:58:59.843Z12/30 Write block 383 len 24576 data: 7 6 6 8 8 9
101502024-02-07T18:58:59.843Z13/30 Flush
101512024-02-07T18:58:59.843Z14/30 Write block 518 len 8192 data: 7 7
101522024-02-07T18:58:59.843Z15/30 Write block 535 len 4096 data: 11
101532024-02-07T18:58:59.843Z16/30 Read block 325 len 36864
101542024-02-07T18:58:59.843Z17/30 Read block 126 len 24576
101552024-02-07T18:58:59.844Z18/30 Read block 243 len 24576
101562024-02-07T18:58:59.844Z19/30 Read block 108 len 40960
101572024-02-07T18:58:59.844Z20/30 Read block 480 len 4096
101582024-02-07T18:58:59.844Z21/30 Write block 344 len 16384 data: 6 9 10 9
101592024-02-07T18:58:59.844Z22/30 Read block 378 len 40960
101602024-02-07T18:58:59.844Z23/30 Read block 139 len 20480
101612024-02-07T18:58:59.844Z24/30 Write block 117 len 20480 data: 9 10 11 10 10
101622024-02-07T18:58:59.844Z25/30 Read block 57 len 12288
101632024-02-07T18:58:59.844Z26/30 Flush
101642024-02-07T18:58:59.860Z27/30 Write block 201 len 16384 data: 5 8 6 8
101652024-02-07T18:58:59.860Z28/30 Write block 514 len 12288 data: 9 8 7
101662024-02-07T18:58:59.860Z29/30 Write block 457 len 20480 data: 8 10 9 8 7
101672024-02-07T18:58:59.860Z30/30 Read block 262 len 28672
101682024-02-07T18:58:59.860Z----------------------------------------------------------------
101692024-02-07T18:58:59.860Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:26
101702024-02-07T18:58:59.860ZStates: Active Active Active
101712024-02-07T18:58:59.860ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101722024-02-07T18:58:59.861Z 5 Acked 1004 Write 1 Done Done Done false
101732024-02-07T18:58:59.861Z 6 Acked 1005 Write 2 Done Done Done false
101742024-02-07T18:58:59.861Z 7 Acked 1006 Write 4 Done Done Sent false
101752024-02-07T18:58:59.861Z 8 Acked 1007 Write 10 Done Done Sent false
101762024-02-07T18:58:59.861Z 9 Acked 1008 Read 9 Done Done Sent false
101772024-02-07T18:58:59.861Z 10 Acked 1009 Write 4 Done Done Sent false
101782024-02-07T18:58:59.861Z 11 Acked 1010 Write 1 Done Done Sent false
101792024-02-07T18:58:59.861Z 12 Acked 1011 Write 6 Done Done Sent false
101802024-02-07T18:58:59.861Z 13 Acked 1012 Flush 0 Done Done Sent false
101812024-02-07T18:58:59.861Z 14 Acked 1013 Write 2 Done Done Sent false
101822024-02-07T18:58:59.862Z 15 Acked 1014 Write 1 Done Done Sent false
101832024-02-07T18:58:59.862Z 16 Acked 1015 Read 9 Done Done Sent false
101842024-02-07T18:58:59.862Z 17 Acked 1016 Read 6 Done Done Sent false
101852024-02-07T18:58:59.862Z 18 Acked 1017 Read 6 Done Done Sent false
101862024-02-07T18:58:59.862Z 19 Acked 1018 Read 10 Done Done Sent false
101872024-02-07T18:58:59.862Z 20 Acked 1019 Read 1 Done Done Sent false
101882024-02-07T18:58:59.862Z 21 Acked 1020 Write 4 Done Done Sent false
101892024-02-07T18:58:59.862Z 22 Acked 1021 Read 10 Done Done Sent false
101902024-02-07T18:58:59.862Z 23 Acked 1022 Read 5 Done Done Sent false
101912024-02-07T18:58:59.863Z 24 Acked 1023 Write 5 Done Done Sent false
101922024-02-07T18:58:59.863Z 25 Acked 1024 Read 3 Done Done Sent false
101932024-02-07T18:58:59.863Z 26 Acked 1025 Flush 0 Done Done Sent false
101942024-02-07T18:58:59.863Z 27 Acked 1026 Write 4 Done Done Sent false
101952024-02-07T18:58:59.863Z 28 Acked 1027 Write 3 Done Done Sent false
101962024-02-07T18:58:59.863Z 29 Acked 1028 Write 5 Done Done Sent false
101972024-02-07T18:58:59.863Z 30 Acked 1029 Read 7 Done Sent Sent false
101982024-02-07T18:58:59.863Z STATES DS:0 DS:1 DS:2 TOTAL
101992024-02-07T18:58:59.863Z New 0 0 0 0
102002024-02-07T18:58:59.864Z Sent 0 1 24 25
102012024-02-07T18:58:59.864Z Done 26 25 2 53
102022024-02-07T18:58:59.864Z Skipped 0 0 0 0
102032024-02-07T18:58:59.864Z Error 0 0 0 0
102042024-02-07T18:58:59.864ZLast Flush: 1025 1025 1003
102052024-02-07T18:58:59.864ZDownstairs last five completed: 1003 1002 1001 1000
102062024-02-07T18:58:59.864ZUpstairs last five completed: 30 29 28 27 26
102072024-02-07T18:58:59.864Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:58:59.07253951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1503}
102082024-02-07T18:58:59.864ZWrote out file "/tmp/test_repair_verify.data"
102092024-02-07T18:58:59.864Z
102102024-02-07T18:58:59.864ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102112024-02-07T18:58:59.882ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102122024-02-07T18:58:59.882Z 0 000-019 88 88 88 143 143 143 F F F
102132024-02-07T18:58:59.882Z 1 020-039 88 88 88 145 145 145 F F F
102142024-02-07T18:58:59.882Z 2 040-059 92 92 92 151 151 151 F F F
102152024-02-07T18:58:59.882Z 3 060-079 91 91 91 150 150 150 F F F
102162024-02-07T18:58:59.882Z 4 080-099 86 86 86 141 141 141 F F F
102172024-02-07T18:58:59.883Z 5 100-119 92 92 90 153 153 147 F F F <---
102182024-02-07T18:58:59.883Z 6 120-139 92 92 84 153 153 138 F F F <---
102192024-02-07T18:58:59.883Z 7 140-159 90 90 90 148 148 148 F F F
102202024-02-07T18:58:59.883Z 8 160-179 89 89 89 146 146 146 F F F
102212024-02-07T18:58:59.883Z 9 180-199 89 89 89 146 146 146 F F F
102222024-02-07T18:58:59.883Z 10 200-219 92 92 89 152 152 146 T T F <---
102232024-02-07T18:58:59.883Z 11 220-239 89 89 89 146 146 146 F F F
102242024-02-07T18:58:59.884Z 12 240-259 88 88 88 143 143 143 F F F
102252024-02-07T18:58:59.884Z 13 260-279 87 87 87 142 142 142 F F F
102262024-02-07T18:58:59.884Z 14 280-299 90 90 90 148 148 148 F F F
102272024-02-07T18:58:59.884Z 15 300-319 88 88 88 145 145 145 F F F
102282024-02-07T18:58:59.884Z 16 320-339 92 92 90 152 152 148 F F F <---
102292024-02-07T18:58:59.884Z 17 340-359 92 92 86 153 153 140 F F T <---
102302024-02-07T18:58:59.885Z 18 360-379 92 92 88 152 152 144 F F T <---
102312024-02-07T18:58:59.885Z 19 380-399 92 92 91 152 152 150 F F F <---
102322024-02-07T18:58:59.885Z 20 400-419 91 91 91 150 150 150 F F F
102332024-02-07T18:58:59.885Z 21 420-439 92 92 92 151 151 151 F F F
102342024-02-07T18:58:59.885Z 22 440-459 92 92 92 151 151 151 T T F <---
102352024-02-07T18:58:59.885Z 23 460-479 90 90 90 149 149 149 T T F <---
102362024-02-07T18:58:59.885Z 24 480-499 84 84 84 138 138 138 F F F
102372024-02-07T18:58:59.886Z 25 500-519 92 92 90 153 153 149 T T F <---
102382024-02-07T18:58:59.886Z 26 520-539 92 92 80 153 153 130 F F F <---
102392024-02-07T18:58:59.886Z 27 540-559 92 92 90 152 152 149 F F F <---
102402024-02-07T18:58:59.886Z 28 560-579 92 92 90 152 152 149 F F F <---
102412024-02-07T18:58:59.886Z 29 580-599 86 86 86 140 140 140 F F F
102422024-02-07T18:58:59.886ZMax gen: 92, Max flush: 153
102432024-02-07T18:58:59.886ZError: Difference in extent metadata found!
102442024-02-07T18:58:59.887ZA Difference in extent metadata is expected here
102452024-02-07T18:58:59.887ZOn loop 46
102462024-02-07T18:58:59.887Z
102472024-02-07T18:58:59.887ZVerifying data now
102482024-02-07T18:59:05.513ZNo change after verify
102492024-02-07T18:59:05.513ZLoop: 46 Downstairs dump after verify (and repair):
102502024-02-07T18:59:05.528ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102512024-02-07T18:59:05.528Z 0 000-019 88 88 88 143 143 143 F F F
102522024-02-07T18:59:05.528Z 1 020-039 88 88 88 145 145 145 F F F
102532024-02-07T18:59:05.528Z 2 040-059 92 92 92 151 151 151 F F F
102542024-02-07T18:59:05.528Z 3 060-079 91 91 91 150 150 150 F F F
102552024-02-07T18:59:05.528Z 4 080-099 86 86 86 141 141 141 F F F
102562024-02-07T18:59:05.528Z 5 100-119 92 92 92 153 153 153 F F F
102572024-02-07T18:59:05.536Z 6 120-139 92 92 92 153 153 153 F F F
102582024-02-07T18:59:05.536Z 7 140-159 90 90 90 148 148 148 F F F
102592024-02-07T18:59:05.536Z 8 160-179 89 89 89 146 146 146 F F F
102602024-02-07T18:59:05.536Z 9 180-199 89 89 89 146 146 146 F F F
102612024-02-07T18:59:05.536Z 10 200-219 93 93 93 154 154 154 F F F
102622024-02-07T18:59:05.536Z 11 220-239 89 89 89 146 146 146 F F F
102632024-02-07T18:59:05.537Z 12 240-259 88 88 88 143 143 143 F F F
102642024-02-07T18:59:05.537Z 13 260-279 87 87 87 142 142 142 F F F
102652024-02-07T18:59:05.537Z 14 280-299 90 90 90 148 148 148 F F F
102662024-02-07T18:59:05.537Z 15 300-319 88 88 88 145 145 145 F F F
102672024-02-07T18:59:05.537Z 16 320-339 92 92 92 152 152 152 F F F
102682024-02-07T18:59:05.537Z 17 340-359 92 92 92 153 153 153 F F F
102692024-02-07T18:59:05.537Z 18 360-379 92 92 92 152 152 152 F F F
102702024-02-07T18:59:05.537Z 19 380-399 92 92 92 152 152 152 F F F
102712024-02-07T18:59:05.537Z 20 400-419 91 91 91 150 150 150 F F F
102722024-02-07T18:59:05.537Z 21 420-439 92 92 92 151 151 151 F F F
102732024-02-07T18:59:05.537Z 22 440-459 93 93 93 154 154 154 F F F
102742024-02-07T18:59:05.537Z 23 460-479 93 93 93 154 154 154 F F F
102752024-02-07T18:59:05.537Z 24 480-499 84 84 84 138 138 138 F F F
102762024-02-07T18:59:05.537Z 25 500-519 93 93 93 154 154 154 F F F
102772024-02-07T18:59:05.537Z 26 520-539 92 92 92 153 153 153 F F F
102782024-02-07T18:59:05.537Z 27 540-559 92 92 92 152 152 152 F F F
102792024-02-07T18:59:05.537Z 28 560-579 92 92 92 152 152 152 F F F
102802024-02-07T18:59:05.537Z 29 580-599 86 86 86 140 140 140 F F F
102812024-02-07T18:59:05.537ZMax gen: 93, Max flush: 154
102822024-02-07T18:59:05.537Z
102832024-02-07T18:59:05.537ZBegin loop 47
102842024-02-07T18:59:05.537ZDownstairs to restart: 0
102852024-02-07T18:59:05.550Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.784451011Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511}
102862024-02-07T18:59:05.551Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.784700507Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511}
102872024-02-07T18:59:05.551Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.784718815Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511}
102882024-02-07T18:59:05.551Z{"msg":"Crucible e9dd1e97-a487-423d-bcf5-dd509304f0a2 has session id: 650e17ab-a33f-48bf-a31b-743bbea333bc","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.784732263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102892024-02-07T18:59:05.557Z{"msg":"Upstairs opts: Upstairs UUID: e9dd1e97-a487-423d-bcf5-dd509304f0a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.784775547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102902024-02-07T18:59:05.557Z{"msg":"Crucible stats registered with UUID: e9dd1e97-a487-423d-bcf5-dd509304f0a2","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.784801724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102912024-02-07T18:59:05.557ZCrucible runtime is spawned
102922024-02-07T18:59:05.557Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785000967Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511}
102932024-02-07T18:59:05.557Z{"msg":"e9dd1e97-a487-423d-bcf5-dd509304f0a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785038682Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102942024-02-07T18:59:05.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785055779Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102952024-02-07T18:59:05.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785092264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102962024-02-07T18:59:05.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785113031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102972024-02-07T18:59:05.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785137778Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102982024-02-07T18:59:05.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785152666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
102992024-02-07T18:59:05.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785172734Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103002024-02-07T18:59:05.557Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785206809Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103012024-02-07T18:59:05.557Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785222687Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103022024-02-07T18:59:05.557Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785236495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103032024-02-07T18:59:05.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785383745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103042024-02-07T18:59:05.558Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:04.785450506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103052024-02-07T18:59:05.558Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:04.78549446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103062024-02-07T18:59:05.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785519856Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103072024-02-07T18:59:05.558Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:04.785673676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103082024-02-07T18:59:05.558Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785701772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103092024-02-07T18:59:05.558Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.78571666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103102024-02-07T18:59:05.558Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.785850792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103112024-02-07T18:59:05.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.786238639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103122024-02-07T18:59:05.558Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.786279254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103132024-02-07T18:59:05.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.786295192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103142024-02-07T18:59:05.558Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.786318599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103152024-02-07T18:59:05.558Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.786926626Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103162024-02-07T18:59:05.558Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787100303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103172024-02-07T18:59:05.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787502668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103182024-02-07T18:59:05.558Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787550972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103192024-02-07T18:59:05.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787570599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103202024-02-07T18:59:05.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787592596Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103212024-02-07T18:59:05.558Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787607774Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103222024-02-07T18:59:05.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:04.787629381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103232024-02-07T18:59:06.554Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.787285227Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103242024-02-07T18:59:06.555Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.787359188Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103252024-02-07T18:59:06.561Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.788120705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103262024-02-07T18:59:06.561Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.788164919Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103272024-02-07T18:59:06.561Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.788650963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103282024-02-07T18:59:06.562Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789286847Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103292024-02-07T18:59:06.562Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789332151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103302024-02-07T18:59:06.562Z{"msg":"[0]R flush_numbers[0..12]: [143, 145, 151, 150, 141, 153, 153, 148, 146, 146, 154, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789349688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103312024-02-07T18:59:06.562Z{"msg":"[0]R generation[0..12]: [88, 88, 92, 91, 86, 92, 92, 90, 89, 89, 93, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789365136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103322024-02-07T18:59:06.562Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789398162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"Wait for a query_work_queue command to finish before sending IO
103332024-02-07T18:59:06.563Z}
103342024-02-07T18:59:06.563Z{"msg":"[1]R flush_numbers[0..12]: [143, 145, 151, 150, 141, 153, 153, 148, 146, 146, 154, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789438416Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103352024-02-07T18:59:06.563Z{"msg":"[1]R generation[0..12]: [88, 88, 92, 91, 86, 92, 92, 90, 89, 89, 93, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789474462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103362024-02-07T18:59:06.563Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789493129Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103372024-02-07T18:59:06.563Z{"msg":"[2]R flush_numbers[0..12]: [143, 145, 151, 150, 141, 153, 153, 148, 146, 146, 154, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789515076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103382024-02-07T18:59:06.563Z{"msg":"[2]R generation[0..12]: [88, 88, 92, 91, 86, 92, 92, 90, 89, 89, 93, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789529384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103392024-02-07T18:59:06.563Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789563299Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103402024-02-07T18:59:06.564Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789598825Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103412024-02-07T18:59:06.564Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789619292Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103422024-02-07T18:59:06.564Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789643149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103432024-02-07T18:59:06.564Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789667985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103442024-02-07T18:59:06.564Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789702571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103452024-02-07T18:59:06.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789721038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103462024-02-07T18:59:06.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789744055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103472024-02-07T18:59:06.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789769362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103482024-02-07T18:59:06.565Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789804137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103492024-02-07T18:59:06.565Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789821535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103502024-02-07T18:59:06.565Z{"msg":"e9dd1e97-a487-423d-bcf5-dd509304f0a2 is now active with session: 650e17ab-a33f-48bf-a31b-743bbea333bc","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789844121Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103512024-02-07T18:59:06.565Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:05.789867808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511,"session_id":"650e17ab-a33f-48bf-a31b-743bbea333bc"}
103522024-02-07T18:59:06.565Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103532024-02-07T18:59:06.565Z,"time":"2024-02-07T18:59:05.789905153Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511}
103542024-02-07T18:59:06.565ZLoading write count information from file "/tmp/test_repair_verify.data"
103552024-02-07T18:59:06.565ZRun Repair workload
103562024-02-07T18:59:06.565Z01/30 Flush
103572024-02-07T18:59:06.566Z02/30 Write block 465 len 40960 data: 7 8 4 3 4 6 6 5 5 4
103582024-02-07T18:59:06.570Z03/30 Write block 237 len 28672 data: 13 13 13 15 15 16 14
103592024-02-07T18:59:06.571Z04/30 Write block 263 len 8192 data: 10 10
103602024-02-07T18:59:06.571Z05/30 Read block 157 len 12288
103612024-02-07T18:59:06.571Z06/30 Flush
103622024-02-07T18:59:06.571Z07/30 Write block 245 len 8192 data: 9 9
103632024-02-07T18:59:06.571Z08/30 Write block 530 len 4096 data: 7
103642024-02-07T18:59:06.571Z09/30 Read block 433 len 8192
103652024-02-07T18:59:06.572Z10/30 Read block 297 len 32768
103662024-02-07T18:59:06.572Z11/30 Read block 55 len 20480
103672024-02-07T18:59:06.572Z12/30 Read block 177 len 4096
103682024-02-07T18:59:06.572Z13/30 Flush
103692024-02-07T18:59:06.572Z14/30 Write block 100 len 40960 data: 5 5 5 6 9 11 10 11 11 11
103702024-02-07T18:59:06.992Z15/30 Read block 358 len 16384
103712024-02-07T18:59:06.993Z16/30 Write block 457 len 28672 data: 9 11 10 9 8 6 7
103722024-02-07T18:59:06.993Z17/30 Read block 214 len 28672
103732024-02-07T18:59:06.993Z18/30 Write block 347 len 40960 data: 10 9 10 10 9 6 5 8 6 4
103742024-02-07T18:59:06.993Z19/30 Read block 591 len 4096
103752024-02-07T18:59:06.993Z20/30 Read block 187 len 8192
103762024-02-07T18:59:06.993Z21/30 Write block 199 len 28672 data: 7 6 6 9 7 9 10
103772024-02-07T18:59:06.993Z22/30 Read block 379 len 36864
103782024-02-07T18:59:06.993Z23/30 Write block 371 len 24576 data: 10 7 6 5 6 6
103792024-02-07T18:59:06.994Z24/30 Read block 482 len 12288
103802024-02-07T18:59:07.001Z25/30 Read block 386 len 20480
103812024-02-07T18:59:07.001Z26/30 Read block 261 len 40960
103822024-02-07T18:59:07.001Z27/30 Write block 149 len 36864 data: 14 9 9 9 10 10 6 5 4
103832024-02-07T18:59:07.001Z28/30 Write block 479 len 36864 data: 6 6 5 5 7 6 3 3 5
103842024-02-07T18:59:07.001Z29/30 Read block 525 len 4096
103852024-02-07T18:59:07.001Z30/30 Write block 124 len 24576 data: 7 7 6 7 9 9
103862024-02-07T18:59:07.002Z----------------------------------------------------------------
103872024-02-07T18:59:07.002Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:29
103882024-02-07T18:59:07.002ZStates: Active Active Active
103892024-02-07T18:59:07.002ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103902024-02-07T18:59:07.002Z 2 Acked 1001 Write 10 Sent Done Done false
103912024-02-07T18:59:07.002Z 3 Acked 1002 Write 7 Sent Done Done false
103922024-02-07T18:59:07.002Z 4 Acked 1003 Write 2 Sent Done Done false
103932024-02-07T18:59:07.002Z 5 Acked 1004 Read 3 Sent Done Done false
103942024-02-07T18:59:07.002Z 6 Acked 1005 Flush 0 Sent Done Done false
103952024-02-07T18:59:07.002Z 7 Acked 1006 Write 2 Sent Done Done false
103962024-02-07T18:59:07.002Z 8 Acked 1007 Write 1 Sent Done Done false
103972024-02-07T18:59:07.002Z 9 Acked 1008 Read 2 Sent Done Done false
103982024-02-07T18:59:07.002Z 10 Acked 1009 Read 8 Sent Done Done false
103992024-02-07T18:59:07.002Z 11 Acked 1010 Read 5 Sent Done Done false
104002024-02-07T18:59:07.002Z 12 Acked 1011 Read 1 Sent Done Done false
104012024-02-07T18:59:07.003Z 13 Acked 1012 Flush 0 Sent Done Done false
104022024-02-07T18:59:07.003Z 14 Acked 1013 Write 10 Sent Done Done false
104032024-02-07T18:59:07.003Z 15 Acked 1014 Read 4 Sent Done Done false
104042024-02-07T18:59:07.003Z 16 Acked 1015 Write 7 Sent Done Done false
104052024-02-07T18:59:07.003Z 17 Acked 1016 Read 7 Sent Done Done false
104062024-02-07T18:59:07.003Z 18 Acked 1017 Write 10 Sent Done Done false
104072024-02-07T18:59:07.003Z 19 Acked 1018 Read 1 Sent Done Done false
104082024-02-07T18:59:07.003Z 20 Acked 1019 Read 2 Sent Done Done false
104092024-02-07T18:59:07.003Z 21 Acked 1020 Write 7 Sent Done Done false
104102024-02-07T18:59:07.003Z 22 Acked 1021 Read 9 Sent Done Done false
104112024-02-07T18:59:07.003Z 23 Acked 1022 Write 6 Sent Done Done false
104122024-02-07T18:59:07.003Z 24 Acked 1023 Read 3 Sent Done Done false
104132024-02-07T18:59:07.003Z 25 Acked 1024 Read 5 Sent Done Done false
104142024-02-07T18:59:07.003Z 26 Acked 1025 Read 10 Sent Done Done false
104152024-02-07T18:59:07.003Z 27 Acked 1026 Write 9 Sent Done Done false
104162024-02-07T18:59:07.003Z 28 Acked 1027 Write 9 Sent Done Done false
104172024-02-07T18:59:07.003Z 29 Acked 1028 Read 1 Sent Done Done false
104182024-02-07T18:59:07.003Z 30 Acked 1029 Write 6 Sent Sent Sent false
104192024-02-07T18:59:07.003Z STATES DS:0 DS:1 DS:2 TOTAL
104202024-02-07T18:59:07.003Z New 0 0 0 0
104212024-02-07T18:59:07.003Z Sent 29 1 1 31
104222024-02-07T18:59:07.003Z Done 0 28 28 56
104232024-02-07T18:59:07.003Z Skipped 0 0 0 0
104242024-02-07T18:59:07.003Z Error 0 0 0 0
104252024-02-07T18:59:07.003ZLast Flush: 1000 1012 1012
104262024-02-07T18:59:07.004ZDownstairs last five completed: 1000
104272024-02-07T18:59:07.004ZUpstairs last five completed: 30 29 28 27 26
104282024-02-07T18:59:07.004Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:05.820493606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1511}
104292024-02-07T18:59:07.004ZWrote out file "/tmp/test_repair_verify.data"
104302024-02-07T18:59:07.004Z
104312024-02-07T18:59:07.004ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104322024-02-07T18:59:07.004ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104332024-02-07T18:59:07.004Z 0 000-019 88 88 88 143 143 143 F F F
104342024-02-07T18:59:07.004Z 1 020-039 88 88 88 145 145 145 F F F
104352024-02-07T18:59:07.004Z 2 040-059 92 92 92 151 151 151 F F F
104362024-02-07T18:59:07.004Z 3 060-079 91 91 91 150 150 150 F F F
104372024-02-07T18:59:07.004Z 4 080-099 86 86 86 141 141 141 F F F
104382024-02-07T18:59:07.004Z 5 100-119 92 92 92 153 153 153 F T T <---
104392024-02-07T18:59:07.004Z 6 120-139 92 92 92 153 153 153 F T T <---
104402024-02-07T18:59:07.004Z 7 140-159 90 90 90 148 148 148 F T T <---
104412024-02-07T18:59:07.004Z 8 160-179 89 89 89 146 146 146 F F F
104422024-02-07T18:59:07.004Z 9 180-199 89 89 89 146 146 146 F T T <---
104432024-02-07T18:59:07.004Z 10 200-219 93 93 93 154 154 154 F T T <---
104442024-02-07T18:59:07.004Z 11 220-239 89 94 94 146 156 156 F F F <---
104452024-02-07T18:59:07.004Z 12 240-259 88 94 94 143 157 157 F F F <---
104462024-02-07T18:59:07.004Z 13 260-279 87 94 94 142 156 156 F F F <---
104472024-02-07T18:59:07.004Z 14 280-299 90 90 90 148 148 148 F F F
104482024-02-07T18:59:07.004Z 15 300-319 88 88 88 145 145 145 F F F
104492024-02-07T18:59:07.004Z 16 320-339 92 92 92 152 152 152 F F F
104502024-02-07T18:59:07.005Z 17 340-359 92 92 92 153 153 153 F T T <---
104512024-02-07T18:59:07.005Z 18 360-379 92 92 92 152 152 152 F T T <---
104522024-02-07T18:59:07.005Z 19 380-399 92 92 92 152 152 152 F F F
104532024-02-07T18:59:07.005Z 20 400-419 91 91 91 150 150 150 F F F
104542024-02-07T18:59:07.005Z 21 420-439 92 92 92 151 151 151 F F F
104552024-02-07T18:59:07.005Z 22 440-459 93 93 93 154 154 154 F T T <---
104562024-02-07T18:59:07.005Z 23 460-479 93 94 94 154 156 156 F T T <---
104572024-02-07T18:59:07.005Z 24 480-499 84 84 84 138 138 138 F T T <---
104582024-02-07T18:59:07.005Z 25 500-519 93 93 93 154 154 154 F F F
104592024-02-07T18:59:07.005Z 26 520-539 92 94 94 153 157 157 F F F <---
104602024-02-07T18:59:07.005Z 27 540-559 92 92 92 152 152 152 F F F
104612024-02-07T18:59:07.005Z 28 560-579 92 92 92 152 152 152 F F F
104622024-02-07T18:59:07.005Z 29 580-599 86 86 86 140 140 140 F F F
104632024-02-07T18:59:07.005ZMax gen: 94, Max flush: 157
104642024-02-07T18:59:07.005ZError: Difference in extent metadata found!
104652024-02-07T18:59:07.005ZA Difference in extent metadata is expected here
104662024-02-07T18:59:07.005ZOn loop 47
104672024-02-07T18:59:07.005Z
104682024-02-07T18:59:07.005ZVerifying data now
104692024-02-07T18:59:11.304ZNo change after verify
104702024-02-07T18:59:11.304ZLoop: 47 Downstairs dump after verify (and repair):
104712024-02-07T18:59:11.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104722024-02-07T18:59:11.320Z 0 000-019 88 88 88 143 143 143 F F F
104732024-02-07T18:59:11.320Z 1 020-039 88 88 88 145 145 145 F F F
104742024-02-07T18:59:11.321Z 2 040-059 92 92 92 151 151 151 F F F
104752024-02-07T18:59:11.321Z 3 060-079 91 91 91 150 150 150 F F F
104762024-02-07T18:59:11.321Z 4 080-099 86 86 86 141 141 141 F F F
104772024-02-07T18:59:11.321Z 5 100-119 95 95 95 158 158 158 F F F
104782024-02-07T18:59:11.321Z 6 120-139 95 95 95 158 158 158 F F F
104792024-02-07T18:59:11.321Z 7 140-159 95 95 95 158 158 158 F F F
104802024-02-07T18:59:11.321Z 8 160-179 89 89 89 146 146 146 F F F
104812024-02-07T18:59:11.321Z 9 180-199 95 95 95 158 158 158 F F F
104822024-02-07T18:59:11.321Z 10 200-219 95 95 95 158 158 158 F F F
104832024-02-07T18:59:11.321Z 11 220-239 94 94 94 156 156 156 F F F
104842024-02-07T18:59:11.322Z 12 240-259 94 94 94 157 157 157 F F F
104852024-02-07T18:59:11.322Z 13 260-279 94 94 94 156 156 156 F F F
104862024-02-07T18:59:11.332Z 14 280-299 90 90 90 148 148 148 F F F
104872024-02-07T18:59:11.332Z 15 300-319 88 88 88 145 145 145 F F F
104882024-02-07T18:59:11.332Z 16 320-339 92 92 92 152 152 152 F F F
104892024-02-07T18:59:11.332Z 17 340-359 95 95 95 158 158 158 F F F
104902024-02-07T18:59:11.332Z 18 360-379 95 95 95 158 158 158 F F F
104912024-02-07T18:59:11.332Z 19 380-399 92 92 92 152 152 152 F F F
104922024-02-07T18:59:11.333Z 20 400-419 91 91 91 150 150 150 F F F
104932024-02-07T18:59:11.333Z 21 420-439 92 92 92 151 151 151 F F F
104942024-02-07T18:59:11.333Z 22 440-459 95 95 95 158 158 158 F F F
104952024-02-07T18:59:11.333Z 23 460-479 95 95 95 158 158 158 F F F
104962024-02-07T18:59:11.333Z 24 480-499 95 95 95 158 158 158 F F F
104972024-02-07T18:59:11.333Z 25 500-519 93 93 93 154 154 154 F F F
104982024-02-07T18:59:11.333Z 26 520-539 94 94 94 157 157 157 F F F
104992024-02-07T18:59:11.333Z 27 540-559 92 92 92 152 152 152 F F F
105002024-02-07T18:59:11.333Z 28 560-579 92 92 92 152 152 152 F F F
105012024-02-07T18:59:11.334Z 29 580-599 86 86 86 140 140 140 F F F
105022024-02-07T18:59:11.334ZMax gen: 95, Max flush: 158
105032024-02-07T18:59:11.334Z
105042024-02-07T18:59:11.334ZBegin loop 48
105052024-02-07T18:59:11.334ZDownstairs to restart: 1
105062024-02-07T18:59:11.353Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.569985645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519}
105072024-02-07T18:59:11.353Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570254319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519}
105082024-02-07T18:59:11.353Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570272687Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519}
105092024-02-07T18:59:11.353Z{"msg":"Crucible c786ce09-b8ca-4a96-b095-e3ba5013e8fe has session id: 22514978-9898-46d4-a2bb-88f442ec406b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570290214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105102024-02-07T18:59:11.354Z{"msg":"Upstairs opts: Upstairs UUID: c786ce09-b8ca-4a96-b095-e3ba5013e8fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
105112024-02-07T18:59:11.354Z,"time":"2024-02-07T18:59:10.570336858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105122024-02-07T18:59:11.354Z{"msg":"Crucible stats registered with UUID: c786ce09-b8ca-4a96-b095-e3ba5013e8fe","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570368744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105132024-02-07T18:59:11.354Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570440394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519}
105142024-02-07T18:59:11.355Z{"msg":"c786ce09-b8ca-4a96-b095-e3ba5013e8fe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570598073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105152024-02-07T18:59:11.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570637477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105162024-02-07T18:59:11.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570667223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105172024-02-07T18:59:11.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570708248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105182024-02-07T18:59:11.365Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570729705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105192024-02-07T18:59:11.365Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570751922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105202024-02-07T18:59:11.365Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.57076625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105212024-02-07T18:59:11.365Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570785157Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105222024-02-07T18:59:11.365Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570827942Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105232024-02-07T18:59:11.365Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.570870796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105242024-02-07T18:59:11.366Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.57091368Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105252024-02-07T18:59:11.366Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.571122962Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105262024-02-07T18:59:11.366Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.571162456Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105272024-02-07T18:59:11.366Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.571607446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105282024-02-07T18:59:11.367Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.571647001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105292024-02-07T18:59:11.367Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.571673437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105302024-02-07T18:59:11.367Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.571709232Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105312024-02-07T18:59:11.367Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.572277186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105322024-02-07T18:59:11.367Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.572329199Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105332024-02-07T18:59:11.411Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.572953144Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105342024-02-07T18:59:11.411Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.572990589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105352024-02-07T18:59:11.411Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573010636Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105362024-02-07T18:59:11.411Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573033503Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105372024-02-07T18:59:11.411Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105382024-02-07T18:59:11.411Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573081147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105392024-02-07T18:59:11.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573109783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105402024-02-07T18:59:11.412Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573124471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105412024-02-07T18:59:11.412Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.573566071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105422024-02-07T18:59:11.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574062094Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105432024-02-07T18:59:11.412Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574103308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105442024-02-07T18:59:11.412Z{"msg":"[0]R flush_numbers[0..12]: [143, 145, 151, 150, 141, 158, 158, 158, 146, 158, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574135324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105452024-02-07T18:59:11.412Z{"msg":"[0]R generation[0..12]: [88, 88, 92, 91, 86, 95, 95, 95, 89, 95, 95, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574157931Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105462024-02-07T18:59:11.412Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574173379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105472024-02-07T18:59:11.413Z{"msg":"[1]R flush_numbers[0..12]: [143, 145, 151, 150, 141, 158, 158, 158, 146, 158, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574194086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105482024-02-07T18:59:11.413Z{"msg":"[1]R generation[0..12]: [88, 88, 92, 91, 86, 95, 95, 95, 89, 95, 95, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574208194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105492024-02-07T18:59:11.413Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.57424076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105502024-02-07T18:59:11.413Z{"msg":"[2]R flush_numbers[0..12]: [143, 145, 151, 150, 141, 158, 158, 158, 146, 158, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574272136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105512024-02-07T18:59:11.413Z{"msg":"Wait for a query_work_queue command to finish before sending IO
105522024-02-07T18:59:11.413Z[2]R generation[0..12]: [88, 88, 92, 91, 86, 95, 95, 95, 89, 95, 95, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574290473Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105532024-02-07T18:59:11.413Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574305501Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105542024-02-07T18:59:11.413Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574325858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105552024-02-07T18:59:11.413Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574340147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105562024-02-07T18:59:11.414Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574359814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105572024-02-07T18:59:11.414Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574373502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105582024-02-07T18:59:11.414Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574393379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105592024-02-07T18:59:11.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574405808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105602024-02-07T18:59:11.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574419256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105612024-02-07T18:59:11.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574441003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105622024-02-07T18:59:11.414Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574454761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105632024-02-07T18:59:11.414Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574467379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105642024-02-07T18:59:11.415Z{"msg":"c786ce09-b8ca-4a96-b095-e3ba5013e8fe is now active with session: 22514978-9898-46d4-a2bb-88f442ec406b","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574481008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105652024-02-07T18:59:11.415Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574493096Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519,"session_id":"22514978-9898-46d4-a2bb-88f442ec406b"}
105662024-02-07T18:59:11.415Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:10.574506084Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519}
105672024-02-07T18:59:11.415ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105682024-02-07T18:59:11.415ZLoading write count information from file "/tmp/test_repair_verify.data"
105692024-02-07T18:59:11.415ZRun Repair workload
105702024-02-07T18:59:11.415Z01/30 Write block 370 len 36864 data: 10 11 8 7 6 7 7 6 7
105712024-02-07T18:59:11.416Z02/30 Write block 474 len 20480 data: 5 6 8 6 6
105722024-02-07T18:59:11.416Z03/30 Read block 530 len 4096
105732024-02-07T18:59:11.416Z04/30 Flush
105742024-02-07T18:59:11.416Z05/30 Write block 315 len 12288 data: 12 14 11
105752024-02-07T18:59:11.416Z06/30 Write block 113 len 24576 data: 6 4 5 7 10 11
105762024-02-07T18:59:11.416Z07/30 Write block 194 len 12288 data: 9 7 8
105772024-02-07T18:59:11.416Z08/30 Flush
105782024-02-07T18:59:11.416Z09/30 Write block 236 len 28672 data: 14 14 14 14 16 16 17
105792024-02-07T18:59:11.416Z10/30 Write block 5 len 20480 data: 8 7 10 11 12
105802024-02-07T18:59:11.417Z11/30 Write block 485 len 8192 data: 4 4
105812024-02-07T18:59:11.417Z12/30 Write block 313 len 4096 data: 8
105822024-02-07T18:59:11.417Z13/30 Read block 546 len 8192
105832024-02-07T18:59:11.418Z14/30 Read block 412 len 40960
105842024-02-07T18:59:11.418Z15/30 Read block 216 len 4096
105852024-02-07T18:59:11.418Z16/30 Write block 594 len 12288 data: 7 5 3
105862024-02-07T18:59:11.418Z17/30 Flush
105872024-02-07T18:59:11.418Z18/30 Flush
105882024-02-07T18:59:11.418Z19/30 Write block 255 len 16384 data: 6 7 6 6
105892024-02-07T18:59:11.418Z20/30 Write block 583 len 36864 data: 6 6 7 8 7 8 8 8 9
105902024-02-07T18:59:11.419Z21/30 Read block 579 len 4096
105912024-02-07T18:59:11.419Z22/30 Read block 588 len 4096
105922024-02-07T18:59:11.419Z23/30 Read block 198 len 16384
105932024-02-07T18:59:11.419Z24/30 Write block 197 len 16384 data: 8 10 8 7
105942024-02-07T18:59:11.419Z25/30 Read block 488 len 8192
105952024-02-07T18:59:11.419Z26/30 Read block 503 len 28672
105962024-02-07T18:59:11.420Z27/30 Write block 383 len 28672 data: 8 7 7 9 9 10 10
105972024-02-07T18:59:11.420Z28/30 Read block 484 len 36864
105982024-02-07T18:59:11.420Z29/30 Write block 235 len 20480 data: 16 15 15 15 15
105992024-02-07T18:59:11.420Z30/30 Write block 469 len 28672 data: 5 7 7 6 6 6 7
106002024-02-07T18:59:11.421Z----------------------------------------------------------------
106012024-02-07T18:59:11.421Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:22
106022024-02-07T18:59:11.421ZStates: Active Active Active
106032024-02-07T18:59:11.421ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106042024-02-07T18:59:11.421Z 9 Acked 1008 Write 7 Done Done Done false
106052024-02-07T18:59:11.421Z 10 Acked 1009 Write 5 Done Done Done false
106062024-02-07T18:59:11.421Z 11 Acked 1010 Write 2 Done Sent Done false
106072024-02-07T18:59:11.421Z 12 Acked 1011 Write 1 Done Sent Done false
106082024-02-07T18:59:11.421Z 13 Acked 1012 Read 2 Done Sent Done false
106092024-02-07T18:59:11.422Z 14 Acked 1013 Read 10 Done Sent Done false
106102024-02-07T18:59:11.422Z 15 Acked 1014 Read 1 Done Sent Done false
106112024-02-07T18:59:11.422Z 16 Acked 1015 Write 3 Done Sent Done false
106122024-02-07T18:59:11.422Z 17 Acked 1016 Flush 0 Done Sent Done false
106132024-02-07T18:59:11.422Z 18 Acked 1017 Flush 0 Done Sent Done false
106142024-02-07T18:59:11.422Z 19 Acked 1018 Write 4 Done Sent Done false
106152024-02-07T18:59:11.422Z 20 Acked 1019 Write 9 Done Sent Done false
106162024-02-07T18:59:11.422Z 21 Acked 1020 Read 1 Done Sent Done false
106172024-02-07T18:59:11.422Z 22 Acked 1021 Read 1 Done Sent Done false
106182024-02-07T18:59:11.423Z 23 Acked 1022 Read 4 Done Sent Done false
106192024-02-07T18:59:11.423Z 24 Acked 1023 Write 4 Done Sent Done false
106202024-02-07T18:59:11.423Z 25 Acked 1024 Read 2 Done Sent Done false
106212024-02-07T18:59:11.423Z 26 Acked 1025 Read 7 Done Sent Done false
106222024-02-07T18:59:11.423Z 27 Acked 1026 Write 7 Done Sent Done false
106232024-02-07T18:59:11.423Z 28 Acked 1027 Read 9 Done Sent Done false
106242024-02-07T18:59:11.424Z 29 Acked 1028 Write 5 Done Sent Sent false
106252024-02-07T18:59:11.424Z 30 Acked 1029 Write 7 Sent Sent Sent false
106262024-02-07T18:59:11.424Z STATES DS:0 DS:1 DS:2 TOTAL
106272024-02-07T18:59:11.424Z New 0 0 0 0
106282024-02-07T18:59:11.424Z Sent 1 20 2 23
106292024-02-07T18:59:11.424Z Done 21 2 20 43
106302024-02-07T18:59:11.424Z Skipped 0 0 0 0
106312024-02-07T18:59:11.424Z Error 0 0 0 0
106322024-02-07T18:59:11.425ZLast Flush: 1017 1007 1017
106332024-02-07T18:59:11.438ZDownstairs last five completed: 1007 1006 1005 1004 1003
106342024-02-07T18:59:11.438ZUpstairs last five completed: 30 29 28 27 26
106352024-02-07T18:59:11.438Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:10.603455439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1519}
106362024-02-07T18:59:11.438ZWrote out file "/tmp/test_repair_verify.data"
106372024-02-07T18:59:11.438Z
106382024-02-07T18:59:11.438ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106392024-02-07T18:59:11.438ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106402024-02-07T18:59:11.439Z 0 000-019 96 88 96 161 143 161 F T F <---
106412024-02-07T18:59:11.439Z 1 020-039 88 88 88 145 145 145 F F F
106422024-02-07T18:59:11.439Z 2 040-059 92 92 92 151 151 151 F F F
106432024-02-07T18:59:11.439Z 3 060-079 91 91 91 150 150 150 F F F
106442024-02-07T18:59:11.439Z 4 080-099 86 86 86 141 141 141 F F F
106452024-02-07T18:59:11.439Z 5 100-119 96 96 96 160 160 160 F F F
106462024-02-07T18:59:11.439Z 6 120-139 95 95 95 158 158 158 F F F
106472024-02-07T18:59:11.439Z 7 140-159 95 95 95 158 158 158 F F F
106482024-02-07T18:59:11.439Z 8 160-179 89 89 89 146 146 146 F F F
106492024-02-07T18:59:11.440Z 9 180-199 96 96 96 160 160 160 T F T <---
106502024-02-07T18:59:11.440Z 10 200-219 95 95 95 158 158 158 T F T <---
106512024-02-07T18:59:11.440Z 11 220-239 96 94 96 161 156 161 T T T <---
106522024-02-07T18:59:11.440Z 12 240-259 96 94 96 161 157 161 T T T <---
106532024-02-07T18:59:11.440Z 13 260-279 94 94 94 156 156 156 F F F
106542024-02-07T18:59:11.440Z 14 280-299 90 90 90 148 148 148 F F F
106552024-02-07T18:59:11.440Z 15 300-319 96 96 96 161 160 161 F F F <---
106562024-02-07T18:59:11.440Z 16 320-339 92 92 92 152 152 152 F F F
106572024-02-07T18:59:11.440Z 17 340-359 95 95 95 158 158 158 F F F
106582024-02-07T18:59:11.440Z 18 360-379 96 96 96 159 159 159 F F F
106592024-02-07T18:59:11.441Z 19 380-399 92 92 92 152 152 152 T F T <---
106602024-02-07T18:59:11.441Z 20 400-419 91 91 91 150 150 150 F F F
106612024-02-07T18:59:11.441Z 21 420-439 92 92 92 151 151 151 F F F
106622024-02-07T18:59:11.441Z 22 440-459 95 95 95 158 158 158 F F F
106632024-02-07T18:59:11.441Z 23 460-479 96 96 96 159 159 159 T F T <---
106642024-02-07T18:59:11.441Z 24 480-499 96 95 96 161 158 161 F F F <---
106652024-02-07T18:59:11.441Z 25 500-519 93 93 93 154 154 154 F F F
106662024-02-07T18:59:11.441Z 26 520-539 94 94 94 157 157 157 F F F
106672024-02-07T18:59:11.442Z 27 540-559 92 92 92 152 152 152 F F F
106682024-02-07T18:59:11.442Z 28 560-579 92 92 92 152 152 152 F F F
106692024-02-07T18:59:11.442Z 29 580-599 96 86 96 161 140 161 T F T <---
106702024-02-07T18:59:11.442ZMax gen: 96, Max flush: 161
106712024-02-07T18:59:11.442ZError: Difference in extent metadata found!
106722024-02-07T18:59:11.442ZA Difference in extent metadata is expected here
106732024-02-07T18:59:11.442ZOn loop 48
106742024-02-07T18:59:11.442Z
106752024-02-07T18:59:11.443ZVerifying data now
106762024-02-07T18:59:16.927ZNo change after verify
106772024-02-07T18:59:16.927ZLoop: 48 Downstairs dump after verify (and repair):
106782024-02-07T18:59:16.939ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106792024-02-07T18:59:16.939Z 0 000-019 96 96 96 161 161 161 F F F
106802024-02-07T18:59:16.939Z 1 020-039 88 88 88 145 145 145 F F F
106812024-02-07T18:59:16.940Z 2 040-059 92 92 92 151 151 151 F F F
106822024-02-07T18:59:16.940Z 3 060-079 91 91 91 150 150 150 F F F
106832024-02-07T18:59:16.940Z 4 080-099 86 86 86 141 141 141 F F F
106842024-02-07T18:59:16.940Z 5 100-119 96 96 96 160 160 160 F F F
106852024-02-07T18:59:16.941Z 6 120-139 95 95 95 158 158 158 F F F
106862024-02-07T18:59:16.941Z 7 140-159 95 95 95 158 158 158 F F F
106872024-02-07T18:59:16.941Z 8 160-179 89 89 89 146 146 146 F F F
106882024-02-07T18:59:16.941Z 9 180-199 97 97 97 162 162 162 F F F
106892024-02-07T18:59:16.941Z 10 200-219 97 97 97 162 162 162 F F F
106902024-02-07T18:59:16.942Z 11 220-239 97 97 97 162 162 162 F F F
106912024-02-07T18:59:16.942Z 12 240-259 97 97 97 162 162 162 F F F
106922024-02-07T18:59:16.943Z 13 260-279 94 94 94 156 156 156 F F F
106932024-02-07T18:59:16.943Z 14 280-299 90 90 90 148 148 148 F F F
106942024-02-07T18:59:16.943Z 15 300-319 96 96 96 161 161 161 F F F
106952024-02-07T18:59:16.943Z 16 320-339 92 92 92 152 152 152 F F F
106962024-02-07T18:59:16.944Z 17 340-359 95 95 95 158 158 158 F F F
106972024-02-07T18:59:16.952Z 18 360-379 96 96 96 159 159 159 F F F
106982024-02-07T18:59:16.952Z 19 380-399 97 97 97 162 162 162 F F F
106992024-02-07T18:59:16.952Z 20 400-419 91 91 91 150 150 150 F F F
107002024-02-07T18:59:16.952Z 21 420-439 92 92 92 151 151 151 F F F
107012024-02-07T18:59:16.952Z 22 440-459 95 95 95 158 158 158 F F F
107022024-02-07T18:59:16.953Z 23 460-479 97 97 97 162 162 162 F F F
107032024-02-07T18:59:16.953Z 24 480-499 96 96 96 161 161 161 F F F
107042024-02-07T18:59:16.953Z 25 500-519 93 93 93 154 154 154 F F F
107052024-02-07T18:59:16.953Z 26 520-539 94 94 94 157 157 157 F F F
107062024-02-07T18:59:16.953Z 27 540-559 92 92 92 152 152 152 F F F
107072024-02-07T18:59:16.954Z 28 560-579 92 92 92 152 152 152 F F F
107082024-02-07T18:59:16.954Z 29 580-599 97 97 97 162 162 162 F F F
107092024-02-07T18:59:16.954ZMax gen: 97, Max flush: 162
107102024-02-07T18:59:16.954Z
107112024-02-07T18:59:16.955ZBegin loop 49
107122024-02-07T18:59:16.955ZDownstairs to restart: 2
107132024-02-07T18:59:16.963Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.189860624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527}
107142024-02-07T18:59:16.964Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190138847Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527}
107152024-02-07T18:59:16.964Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190158084Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527}
107162024-02-07T18:59:16.964Z{"msg":"Crucible 7c9fb6de-d891-416a-898e-fdef78e13c8a has session id: 1f694621-9331-40a9-9a22-e4de68ba28b9","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190179931Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107172024-02-07T18:59:16.964Z{"msg":"Upstairs opts: Upstairs UUID: 7c9fb6de-d891-416a-898e-fdef78e13c8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190201158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107182024-02-07T18:59:16.964Z{"msg":"Crucible stats registered with UUID: 7c9fb6de-d891-416a-898e-fdef78e13c8a","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190247282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107192024-02-07T18:59:16.965ZCrucible runtime is spawned
107202024-02-07T18:59:16.965Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190464932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527}
107212024-02-07T18:59:16.965Z{"msg":"7c9fb6de-d891-416a-898e-fdef78e13c8a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190592815Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107222024-02-07T18:59:16.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19063466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107232024-02-07T18:59:16.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190672355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107242024-02-07T18:59:16.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190719418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107252024-02-07T18:59:16.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190756343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107262024-02-07T18:59:16.966Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190791828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107272024-02-07T18:59:16.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190819895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107282024-02-07T18:59:16.966Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190845281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107292024-02-07T18:59:16.966Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190888385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107302024-02-07T18:59:16.966Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190917131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107312024-02-07T18:59:16.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190945798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107322024-02-07T18:59:16.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.190973284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107332024-02-07T18:59:16.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.191059782Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107342024-02-07T18:59:16.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.191639284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107352024-02-07T18:59:16.967Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.191676899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107362024-02-07T18:59:16.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.191693516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107372024-02-07T18:59:16.968Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.191727332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107382024-02-07T18:59:16.976Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.192290985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107392024-02-07T18:59:16.976Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.192415209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107402024-02-07T18:59:16.976Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.192843111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107412024-02-07T18:59:16.976Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.192906402Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107422024-02-07T18:59:16.976Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19292767Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107432024-02-07T18:59:16.976Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19299817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107442024-02-07T18:59:16.976Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.193026536Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107452024-02-07T18:59:16.976Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.193041814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107462024-02-07T18:59:16.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.193062521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107472024-02-07T18:59:16.976Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.193077369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107482024-02-07T18:59:16.976Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.193562984Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107492024-02-07T18:59:16.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19403341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107502024-02-07T18:59:16.977Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194079144Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107512024-02-07T18:59:16.977Z{"msg":"[0]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 160, 158, 158, 146, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19410702Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107522024-02-07T18:59:16.977Z{"msg":"[0]R generation[0..12]: [96, 88, 92, 91, 86, 96, 95, 95, 89, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194142975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107532024-02-07T18:59:16.977Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19417933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107542024-02-07T18:59:16.977Z{"msg":"[1]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 160, 158, 158, 146, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194217095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107552024-02-07T18:59:16.977Z{"msg":"Wait for a query_work_queue command to finish before sending IO
107562024-02-07T18:59:16.977Z[1]R generation[0..12]: [96, 88, 92, 91, 86, 96, 95, 95, 89, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19425686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107572024-02-07T18:59:16.977Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194297974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107582024-02-07T18:59:16.977Z{"msg":"[2]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 160, 158, 158, 146, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194338369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107592024-02-07T18:59:16.977Z{"msg":"[2]R generation[0..12]: [96, 88, 92, 91, 86, 96, 95, 95, 89, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194376023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107602024-02-07T18:59:16.977Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194407639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107612024-02-07T18:59:16.977Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194422657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107622024-02-07T18:59:16.977Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194443254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107632024-02-07T18:59:16.977Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194456733Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107642024-02-07T18:59:16.977Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19447667Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107652024-02-07T18:59:16.977Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194490398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107662024-02-07T18:59:16.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194503156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107672024-02-07T18:59:16.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194523384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107682024-02-07T18:59:16.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194537132Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107692024-02-07T18:59:16.977Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194557169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107702024-02-07T18:59:16.977Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194570387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107712024-02-07T18:59:16.977Z{"msg":"7c9fb6de-d891-416a-898e-fdef78e13c8a is now active with session: 1f694621-9331-40a9-9a22-e4de68ba28b9","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194589905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107722024-02-07T18:59:16.978Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.194602433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527,"session_id":"1f694621-9331-40a9-9a22-e4de68ba28b9"}
107732024-02-07T18:59:16.978ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
107742024-02-07T18:59:16.978Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:16.19462522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527}
107752024-02-07T18:59:16.978ZLoading write count information from file "/tmp/test_repair_verify.data"
107762024-02-07T18:59:16.978ZRun Repair workload
107772024-02-07T18:59:16.978Z01/30 Write block 251 len 20480 data: 5 7 6 6 7
107782024-02-07T18:59:16.978Z02/30 Write block 400 len 4096 data: 7
107792024-02-07T18:59:16.978Z03/30 Read block 25 len 4096
107802024-02-07T18:59:16.978Z04/30 Read block 427 len 12288
107812024-02-07T18:59:16.978Z05/30 Read block 89 len 8192
107822024-02-07T18:59:16.978Z06/30 Write block 443 len 32768 data: 11 11 11 9 9 9 7 7
107832024-02-07T18:59:16.978Z07/30 Read block 236 len 36864
107842024-02-07T18:59:16.978Z08/30 Read block 497 len 20480
107852024-02-07T18:59:16.978Z09/30 Write block 263 len 20480 data: 11 11 9 7 7
107862024-02-07T18:59:16.978Z10/30 Flush
107872024-02-07T18:59:16.982Z11/30 Write block 322 len 28672 data: 8 7 6 6 5 4 6
107882024-02-07T18:59:16.982Z12/30 Flush
107892024-02-07T18:59:16.982Z13/30 Write block 395 len 32768 data: 6 6 6 6 6 8 6 8
107902024-02-07T18:59:16.982Z14/30 Read block 57 len 28672
107912024-02-07T18:59:16.982Z15/30 Read block 484 len 28672
107922024-02-07T18:59:16.982Z16/30 Write block 165 len 4096 data: 10
107932024-02-07T18:59:16.982Z17/30 Read block 370 len 8192
107942024-02-07T18:59:16.982Z18/30 Read block 46 len 12288
107952024-02-07T18:59:16.982Z19/30 Read block 549 len 36864
107962024-02-07T18:59:16.982Z20/30 Read block 535 len 36864
107972024-02-07T18:59:16.985Z21/30 Write block 456 len 24576 data: 5 10 12 11 10 9
107982024-02-07T18:59:16.986Z22/30 Read block 247 len 20480
107992024-02-07T18:59:16.986Z23/30 Write block 119 len 8192 data: 12 11
108002024-02-07T18:59:16.986Z24/30 Read block 192 len 8192
108012024-02-07T18:59:16.986Z25/30 Read block 279 len 24576
108022024-02-07T18:59:16.989Z26/30 Read block 82 len 36864
108032024-02-07T18:59:16.989Z27/30 Write block 266 len 32768 data: 8 8 7 7 5 6 6 6
108042024-02-07T18:59:16.989Z28/30 Read block 62 len 24576
108052024-02-07T18:59:16.992Z29/30 Write block 246 len 20480 data: 10 8 8 6 5
108062024-02-07T18:59:16.992Z30/30 Write block 366 len 24576 data: 9 8 9 9 11 12
108072024-02-07T18:59:16.992Z----------------------------------------------------------------
108082024-02-07T18:59:16.992Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:20
108092024-02-07T18:59:16.993ZStates: Active Active Active
108102024-02-07T18:59:16.993ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108112024-02-07T18:59:16.993Z 11 Acked 1010 Write 7 Done Done Sent false
108122024-02-07T18:59:16.993Z 12 Acked 1011 Flush 0 Done Done Sent false
108132024-02-07T18:59:16.993Z 13 Acked 1012 Write 8 Done Done Sent false
108142024-02-07T18:59:16.993Z 14 Acked 1013 Read 7 Done Done Sent false
108152024-02-07T18:59:16.993Z 15 Acked 1014 Read 7 Done Done Sent false
108162024-02-07T18:59:16.993Z 16 Acked 1015 Write 1 Done Done Sent false
108172024-02-07T18:59:16.993Z 17 Acked 1016 Read 2 Done Done Sent false
108182024-02-07T18:59:16.993Z 18 Acked 1017 Read 3 Done Done Sent false
108192024-02-07T18:59:16.993Z 19 Acked 1018 Read 9 Done Done Sent false
108202024-02-07T18:59:16.993Z 20 Acked 1019 Read 9 Done Done Sent false
108212024-02-07T18:59:16.993Z 21 Acked 1020 Write 6 Done Done Sent false
108222024-02-07T18:59:16.993Z 22 Acked 1021 Read 5 Done Done Sent false
108232024-02-07T18:59:16.993Z 23 Acked 1022 Write 2 Done Done Sent false
108242024-02-07T18:59:16.993Z 24 Acked 1023 Read 2 Done Done Sent false
108252024-02-07T18:59:16.993Z 25 Acked 1024 Read 6 Done Done Sent false
108262024-02-07T18:59:16.993Z 26 Acked 1025 Read 9 Done Done Sent false
108272024-02-07T18:59:16.993Z 27 Acked 1026 Write 8 Done Done Sent false
108282024-02-07T18:59:16.993Z 28 Acked 1027 Read 6 Done Done Sent false
108292024-02-07T18:59:16.993Z 29 Acked 1028 Write 5 Done Sent Sent false
108302024-02-07T18:59:16.993Z 30 Acked 1029 Write 6 Sent Sent Sent false
108312024-02-07T18:59:16.993Z STATES DS:0 DS:1 DS:2 TOTAL
108322024-02-07T18:59:16.993Z New 0 0 0 0
108332024-02-07T18:59:16.994Z Sent 1 2 20 23
108342024-02-07T18:59:16.994Z Done 19 18 0 37
108352024-02-07T18:59:16.994Z Skipped 0 0 0 0
108362024-02-07T18:59:16.994Z Error 0 0 0 0
108372024-02-07T18:59:16.994ZLast Flush: 1011 1011 1009
108382024-02-07T18:59:16.994ZDownstairs last five completed: 1009 1008 1007 1006 1005
108392024-02-07T18:59:16.994ZUpstairs last five completed: 30 29 28 27 26
108402024-02-07T18:59:16.994Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:16.223866295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1527}
108412024-02-07T18:59:16.994ZWrote out file "/tmp/test_repair_verify.data"
108422024-02-07T18:59:16.997Z
108432024-02-07T18:59:17.000ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108442024-02-07T18:59:17.019ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108452024-02-07T18:59:17.019Z 0 000-019 96 96 96 161 161 161 F F F
108462024-02-07T18:59:17.019Z 1 020-039 88 88 88 145 145 145 F F F
108472024-02-07T18:59:17.019Z 2 040-059 92 92 92 151 151 151 F F F
108482024-02-07T18:59:17.019Z 3 060-079 91 91 91 150 150 150 F F F
108492024-02-07T18:59:17.019Z 4 080-099 86 86 86 141 141 141 F F F
108502024-02-07T18:59:17.019Z 5 100-119 96 96 96 160 160 160 T T F <---
108512024-02-07T18:59:17.019Z 6 120-139 95 95 95 158 158 158 T T F <---
108522024-02-07T18:59:17.019Z 7 140-159 95 95 95 158 158 158 F F F
108532024-02-07T18:59:17.019Z 8 160-179 89 89 89 146 146 146 T T F <---
108542024-02-07T18:59:17.019Z 9 180-199 97 97 97 162 162 162 F F F
108552024-02-07T18:59:17.026Z 10 200-219 97 97 97 162 162 162 F F F
108562024-02-07T18:59:17.026Z 11 220-239 97 97 97 162 162 162 F F F
108572024-02-07T18:59:17.026Z 12 240-259 98 98 98 163 163 163 T T F <---
108582024-02-07T18:59:17.026Z 13 260-279 98 98 98 163 163 163 T T F <---
108592024-02-07T18:59:17.026Z 14 280-299 90 90 90 148 148 148 F F F
108602024-02-07T18:59:17.026Z 15 300-319 96 96 96 161 161 161 F F F
108612024-02-07T18:59:17.026Z 16 320-339 98 98 92 164 164 152 F F F <---
108622024-02-07T18:59:17.026Z 17 340-359 95 95 95 158 158 158 F F F
108632024-02-07T18:59:17.026Z 18 360-379 96 96 96 159 159 159 T T F <---
108642024-02-07T18:59:17.026Z 19 380-399 97 97 97 162 162 162 T T F <---
108652024-02-07T18:59:17.026Z 20 400-419 98 98 98 163 163 163 T T F <---
108662024-02-07T18:59:17.026Z 21 420-439 92 92 92 151 151 151 F F F
108672024-02-07T18:59:17.026Z 22 440-459 98 98 98 163 163 163 T T F <---
108682024-02-07T18:59:17.026Z 23 460-479 97 97 97 162 162 162 T T F <---
108692024-02-07T18:59:17.026Z 24 480-499 96 96 96 161 161 161 F F F
108702024-02-07T18:59:17.026Z 25 500-519 93 93 93 154 154 154 F F F
108712024-02-07T18:59:17.026Z 26 520-539 94 94 94 157 157 157 F F F
108722024-02-07T18:59:17.026Z 27 540-559 92 92 92 152 152 152 F F F
108732024-02-07T18:59:17.026Z 28 560-579 92 92 92 152 152 152 F F F
108742024-02-07T18:59:17.026Z 29 580-599 97 97 97 162 162 162 F F F
108752024-02-07T18:59:17.026ZMax gen: 98, Max flush: 164
108762024-02-07T18:59:17.026ZError: Difference in extent metadata found!
108772024-02-07T18:59:17.026ZA Difference in extent metadata is expected here
108782024-02-07T18:59:17.027ZOn loop 49
108792024-02-07T18:59:17.027Z
108802024-02-07T18:59:17.034ZVerifying data now
108812024-02-07T18:59:21.603ZNo change after verify
108822024-02-07T18:59:21.603ZLoop: 49 Downstairs dump after verify (and repair):
108832024-02-07T18:59:21.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108842024-02-07T18:59:21.618Z 0 000-019 96 96 96 161 161 161 F F F
108852024-02-07T18:59:21.618Z 1 020-039 88 88 88 145 145 145 F F F
108862024-02-07T18:59:21.618Z 2 040-059 92 92 92 151 151 151 F F F
108872024-02-07T18:59:21.618Z 3 060-079 91 91 91 150 150 150 F F F
108882024-02-07T18:59:21.618Z 4 080-099 86 86 86 141 141 141 F F F
108892024-02-07T18:59:21.618Z 5 100-119 99 99 99 165 165 165 F F F
108902024-02-07T18:59:21.618Z 6 120-139 99 99 99 165 165 165 F F F
108912024-02-07T18:59:22.043Z 7 140-159 95 95 95 158 158 158 F F F
108922024-02-07T18:59:22.043Z 8 160-179 99 99 99 165 165 165 F F F
108932024-02-07T18:59:22.043Z 9 180-199 97 97 97 162 162 162 F F F
108942024-02-07T18:59:22.044Z 10 200-219 97 97 97 162 162 162 F F F
108952024-02-07T18:59:22.044Z 11 220-239 97 97 97 162 162 162 F F F
108962024-02-07T18:59:22.044Z 12 240-259 99 99 99 165 165 165 F F F
108972024-02-07T18:59:22.044Z 13 260-279 99 99 99 165 165 165 F F F
108982024-02-07T18:59:22.045Z 14 280-299 90 90 90 148 148 148 F F F
108992024-02-07T18:59:22.045Z 15 300-319 96 96 96 161 161 161 F F F
109002024-02-07T18:59:22.045Z 16 320-339 98 98 98 164 164 164 F F F
109012024-02-07T18:59:22.045Z 17 340-359 95 95 95 158 158 158 F F F
109022024-02-07T18:59:22.045Z 18 360-379 99 99 99 165 165 165 F F F
109032024-02-07T18:59:22.045Z 19 380-399 99 99 99 165 165 165 F F F
109042024-02-07T18:59:22.046Z 20 400-419 99 99 99 165 165 165 F F F
109052024-02-07T18:59:22.046Z 21 420-439 92 92 92 151 151 151 F F F
109062024-02-07T18:59:22.046Z 22 440-459 99 99 99 165 165 165 F F F
109072024-02-07T18:59:22.046Z 23 460-479 99 99 99 165 165 165 F F F
109082024-02-07T18:59:22.046Z 24 480-499 96 96 96 161 161 161 F F F
109092024-02-07T18:59:22.046Z 25 500-519 93 93 93 154 154 154 F F F
109102024-02-07T18:59:22.046Z 26 520-539 94 94 94 157 157 157 F F F
109112024-02-07T18:59:22.047Z 27 540-559 92 92 92 152 152 152 F F F
109122024-02-07T18:59:22.047Z 28 560-579 92 92 92 152 152 152 F F F
109132024-02-07T18:59:22.047Z 29 580-599 97 97 97 162 162 162 F F F
109142024-02-07T18:59:22.047ZMax gen: 99, Max flush: 165
109152024-02-07T18:59:22.047Z
109162024-02-07T18:59:22.048ZBegin loop 50
109172024-02-07T18:59:22.048ZDownstairs to restart: 1
109182024-02-07T18:59:22.096Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.863519628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535}
109192024-02-07T18:59:22.097Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
109202024-02-07T18:59:22.097Z,"time":"2024-02-07T18:59:20.863856423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535}
109212024-02-07T18:59:22.097Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.863895547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535}
109222024-02-07T18:59:22.097Z{"msg":"Crucible b0acf46e-d8e4-475d-8c02-fec83e1215a4 has session id: 9ecf50ce-71aa-444b-83c1-e72cb64cb4d6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.863945131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109232024-02-07T18:59:22.097Z{"msg":"Upstairs opts: Upstairs UUID: b0acf46e-d8e4-475d-8c02-fec83e1215a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864035738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109242024-02-07T18:59:22.097Z{"msg":"Crucible stats registered with UUID: b0acf46e-d8e4-475d-8c02-fec83e1215a4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864064735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109252024-02-07T18:59:22.097Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.86409546Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535}
109262024-02-07T18:59:22.097Z{"msg":"b0acf46e-d8e4-475d-8c02-fec83e1215a4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864117137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109272024-02-07T18:59:22.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864155982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109282024-02-07T18:59:22.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864182768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109292024-02-07T18:59:22.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864216614Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109302024-02-07T18:59:22.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.86424238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109312024-02-07T18:59:22.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864266957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109322024-02-07T18:59:22.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864295493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109332024-02-07T18:59:22.098Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.86432016Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109342024-02-07T18:59:22.098Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864364424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109352024-02-07T18:59:22.098Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.86439509Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109362024-02-07T18:59:22.098Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864431175Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109372024-02-07T18:59:22.098Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864459711Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109382024-02-07T18:59:22.098Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.864496676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109392024-02-07T18:59:22.099Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.865129011Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109402024-02-07T18:59:22.099Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.865168235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109412024-02-07T18:59:22.099Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.865184513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109422024-02-07T18:59:22.099Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.86520511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109432024-02-07T18:59:22.099Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.865751536Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109442024-02-07T18:59:22.099Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.865879659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109452024-02-07T18:59:22.099Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866206455Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109462024-02-07T18:59:22.099Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.86623777Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109472024-02-07T18:59:22.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866423405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109482024-02-07T18:59:22.099Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866468709Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109492024-02-07T18:59:22.099Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866496446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109502024-02-07T18:59:22.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866531561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109512024-02-07T18:59:22.100Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866560367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109522024-02-07T18:59:22.100Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866593882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109532024-02-07T18:59:22.100Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.866828061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109542024-02-07T18:59:22.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867300847Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109552024-02-07T18:59:22.100Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867341841Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109562024-02-07T18:59:22.101Z{"msg":"[0]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 165, 165, 158, 165, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867358339Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109572024-02-07T18:59:22.101Z{"msg":"[0]R generation[0..12]: [96, 88, 92, 91, 86, 99, 99, 95, 99, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867379956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109582024-02-07T18:59:22.101Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867394714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109592024-02-07T18:59:22.101Z{"msg":"[1]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 165, 165, 158, 165, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867409062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109602024-02-07T18:59:22.101Z{"msg":"[1]R generation[0..12]: [96, 88, 92, 91, 86, 99, 99, 95, 99, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867429799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109612024-02-07T18:59:22.101Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867444397Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109622024-02-07T18:59:22.101Z{"msg":"[2]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 165, 165, 158, 165, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867464935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO
109632024-02-07T18:59:22.101Z:"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109642024-02-07T18:59:22.101Z{"msg":"[2]R generation[0..12]: [96, 88, 92, 91, 86, 99, 99, 95, 99, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867482462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109652024-02-07T18:59:22.101Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867503809Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109662024-02-07T18:59:22.101Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867517637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109672024-02-07T18:59:22.102Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867537775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109682024-02-07T18:59:22.102Z{"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867551113Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109692024-02-07T18:59:22.102Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867564581Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109702024-02-07T18:59:22.102Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867584778Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109712024-02-07T18:59:22.102Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867597737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109722024-02-07T18:59:22.102Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867617604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109732024-02-07T18:59:22.102Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867630992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109742024-02-07T18:59:22.102Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867650819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109752024-02-07T18:59:22.102Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867663518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109762024-02-07T18:59:22.102Z{"msg":"b0acf46e-d8e4-475d-8c02-fec83e1215a4 is now active with session: 9ecf50ce-71aa-444b-83c1-e72cb64cb4d6","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867683365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109772024-02-07T18:59:22.102Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867695653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535,"session_id":"9ecf50ce-71aa-444b-83c1-e72cb64cb4d6"}
109782024-02-07T18:59:22.102Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:20.867715231Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535}
109792024-02-07T18:59:22.102ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
109802024-02-07T18:59:22.103ZLoading write count information from file "/tmp/test_repair_verify.data"
109812024-02-07T18:59:22.103ZRun Repair workload
109822024-02-07T18:59:22.103Z01/30 Flush
109832024-02-07T18:59:22.103Z02/30 Write block 206 len 28672 data: 9 8 10 8 7 8 8
109842024-02-07T18:59:22.103Z03/30 Read block 394 len 32768
109852024-02-07T18:59:22.103Z04/30 Write block 240 len 20480 data: 17 17 18 15 13
109862024-02-07T18:59:22.103Z05/30 Write block 510 len 32768 data: 9 7 10 8 10 9 8 5
109872024-02-07T18:59:22.103Z06/30 Read block 444 len 4096
109882024-02-07T18:59:22.103Z07/30 Read block 74 len 4096
109892024-02-07T18:59:22.103Z08/30 Read block 250 len 20480
109902024-02-07T18:59:22.103Z09/30 Read block 224 len 8192
109912024-02-07T18:59:22.103Z10/30 Read block 257 len 16384
109922024-02-07T18:59:22.104Z11/30 Write block 547 len 16384 data: 8 9 10 9
109932024-02-07T18:59:22.104Z12/30 Flush
109942024-02-07T18:59:22.104Z13/30 Flush
109952024-02-07T18:59:22.104Z14/30 Write block 565 len 28672 data: 10 9 10 9 9 9 8
109962024-02-07T18:59:22.104Z15/30 Write block 500 len 28672 data: 6 5 8 10 10 12 9
109972024-02-07T18:59:22.104Z16/30 Read block 125 len 40960
109982024-02-07T18:59:22.104Z17/30 Flush
109992024-02-07T18:59:22.104Z18/30 Flush
110002024-02-07T18:59:22.104Z19/30 Write block 273 len 12288 data: 7 5 5
110012024-02-07T18:59:22.104Z20/30 Read block 128 len 4096
110022024-02-07T18:59:22.104Z21/30 Flush
110032024-02-07T18:59:22.105Z22/30 Write block 383 len 12288 data: 9 8 8
110042024-02-07T18:59:22.105Z23/30 Write block 145 len 20480 data: 10 12 13 12 15
110052024-02-07T18:59:22.105Z24/30 Flush
110062024-02-07T18:59:22.105Z25/30 Write block 111 len 40960 data: 10 7 7 5 6 8 11 12 13 12
110072024-02-07T18:59:22.105Z26/30 Read block 527 len 24576
110082024-02-07T18:59:22.105Z27/30 Write block 258 len 8192 data: 7 9
110092024-02-07T18:59:22.105Z28/30 Write block 241 len 40960 data: 18 19 16 14 10 11 9 9 7 6
110102024-02-07T18:59:22.105Z29/30 Read block 12 len 8192
110112024-02-07T18:59:22.105Z30/30 Read block 63 len 28672
110122024-02-07T18:59:22.105Z----------------------------------------------------------------
110132024-02-07T18:59:22.106Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:29
110142024-02-07T18:59:22.106ZStates: Active Active Active
110152024-02-07T18:59:22.106ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110162024-02-07T18:59:22.106Z 2 Acked 1001 Write 7 Done Done Done false
110172024-02-07T18:59:22.106Z 3 Acked 1002 Read 8 Done Done Done false
110182024-02-07T18:59:22.128Z 4 Acked 1003 Write 5 Done Done Done false
110192024-02-07T18:59:22.129Z 5 Acked 1004 Write 8 Done Done Done false
110202024-02-07T18:59:22.129Z 6 Acked 1005 Read 1 Done Sent Done false
110212024-02-07T18:59:22.129Z 7 Acked 1006 Read 1 Done Sent Done false
110222024-02-07T18:59:22.129Z 8 Acked 1007 Read 5 Done Sent Done false
110232024-02-07T18:59:22.130Z 9 Acked 1008 Read 2 Done Sent Done false
110242024-02-07T18:59:22.130Z 10 Acked 1009 Read 4 Done Sent Done false
110252024-02-07T18:59:22.130Z 11 Acked 1010 Write 4 Done Sent Done false
110262024-02-07T18:59:22.130Z 12 Acked 1011 Flush 0 Done Sent Done false
110272024-02-07T18:59:22.130Z 13 Acked 1012 Flush 0 Done Sent Done false
110282024-02-07T18:59:22.130Z 14 Acked 1013 Write 7 Done Sent Done false
110292024-02-07T18:59:22.130Z 15 Acked 1014 Write 7 Done Sent Done false
110302024-02-07T18:59:22.131Z 16 Acked 1015 Read 10 Done Sent Done false
110312024-02-07T18:59:22.131Z 17 Acked 1016 Flush 0 Done Sent Done false
110322024-02-07T18:59:22.131Z 18 Acked 1017 Flush 0 Done Sent Done false
110332024-02-07T18:59:22.131Z 19 Acked 1018 Write 3 Done Sent Done false
110342024-02-07T18:59:22.131Z 20 Acked 1019 Read 1 Done Sent Done false
110352024-02-07T18:59:22.131Z 21 Acked 1020 Flush 0 Done Sent Done false
110362024-02-07T18:59:22.132Z 22 Acked 1021 Write 3 Done Sent Done false
110372024-02-07T18:59:22.132Z 23 Acked 1022 Write 5 Done Sent Done false
110382024-02-07T18:59:22.132Z 24 Acked 1023 Flush 0 Done Sent Done false
110392024-02-07T18:59:22.132Z 25 Acked 1024 Write 10 Done Sent Done false
110402024-02-07T18:59:22.132Z 26 Acked 1025 Read 6 Done Sent Done false
110412024-02-07T18:59:22.132Z 27 Acked 1026 Write 2 Done Sent Done false
110422024-02-07T18:59:22.133Z 28 Acked 1027 Write 10 Done Sent Done false
110432024-02-07T18:59:22.133Z 29 Acked 1028 Read 2 Done Sent Done false
110442024-02-07T18:59:22.133Z 30 Acked 1029 Read 7 Done Sent Done false
110452024-02-07T18:59:22.133Z STATES DS:0 DS:1 DS:2 TOTAL
110462024-02-07T18:59:22.133Z New 0 0 0 0
110472024-02-07T18:59:22.133Z Sent 0 25 0 25
110482024-02-07T18:59:22.133Z Done 29 4 29 62
110492024-02-07T18:59:22.134Z Skipped 0 0 0 0
110502024-02-07T18:59:22.134Z Error 0 0 0 0
110512024-02-07T18:59:22.134ZLast Flush: 1023 1000 1023
110522024-02-07T18:59:22.134ZDownstairs last five completed: 1000
110532024-02-07T18:59:22.135ZUpstairs last five completed: 30 29 28 27 26
110542024-02-07T18:59:22.135Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:20.898280557Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1535}
110552024-02-07T18:59:22.135ZWrote out file "/tmp/test_repair_verify.data"
110562024-02-07T18:59:22.135Z
110572024-02-07T18:59:22.135ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
110582024-02-07T18:59:22.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110592024-02-07T18:59:22.136Z 0 000-019 96 96 96 161 161 161 F F F
110602024-02-07T18:59:22.136Z 1 020-039 88 88 88 145 145 145 F F F
110612024-02-07T18:59:22.136Z 2 040-059 92 92 92 151 151 151 F F F
110622024-02-07T18:59:22.136Z 3 060-079 91 91 91 150 150 150 F F F
110632024-02-07T18:59:22.136Z 4 080-099 86 86 86 141 141 141 F F F
110642024-02-07T18:59:22.137Z 5 100-119 99 99 99 165 165 165 T F T <---
110652024-02-07T18:59:22.137Z 6 120-139 99 99 99 165 165 165 T F T <---
110662024-02-07T18:59:22.137Z 7 140-159 100 95 100 172 158 172 F F F <---
110672024-02-07T18:59:22.137Z 8 160-179 99 99 99 165 165 165 F F F
110682024-02-07T18:59:22.137Z 9 180-199 97 97 97 162 162 162 F F F
110692024-02-07T18:59:22.137Z 10 200-219 100 97 100 167 162 167 F T F <---
110702024-02-07T18:59:22.137Z 11 220-239 97 97 97 162 162 162 F F F
110712024-02-07T18:59:22.138Z 12 240-259 100 99 100 167 165 167 T T T <---
110722024-02-07T18:59:22.138Z 13 260-279 100 99 100 171 165 171 F F F <---
110732024-02-07T18:59:22.138Z 14 280-299 90 90 90 148 148 148 F F F
110742024-02-07T18:59:22.138Z 15 300-319 96 96 96 161 161 161 F F F
110752024-02-07T18:59:22.138Z 16 320-339 98 98 98 164 164 164 F F F
110762024-02-07T18:59:22.138Z 17 340-359 95 95 95 158 158 158 F F F
110772024-02-07T18:59:22.138Z 18 360-379 99 99 99 165 165 165 F F F
110782024-02-07T18:59:22.139Z 19 380-399 100 99 100 172 165 172 F F F <---
110792024-02-07T18:59:22.139Z 20 400-419 99 99 99 165 165 165 F F F
110802024-02-07T18:59:22.139Z 21 420-439 92 92 92 151 151 151 F F F
110812024-02-07T18:59:22.139Z 22 440-459 99 99 99 165 165 165 F F F
110822024-02-07T18:59:22.139Z 23 460-479 99 99 99 165 165 165 F F F
110832024-02-07T18:59:22.139Z 24 480-499 96 96 96 161 161 161 F F F
110842024-02-07T18:59:22.140Z 25 500-519 100 93 100 169 154 169 F T F <---
110852024-02-07T18:59:22.140Z 26 520-539 94 94 94 157 157 157 F F F
110862024-02-07T18:59:22.140Z 27 540-559 100 92 100 167 152 167 F F F <---
110872024-02-07T18:59:22.140Z 28 560-579 100 92 100 169 152 169 F F F <---
110882024-02-07T18:59:22.140Z 29 580-599 97 97 97 162 162 162 F F F
110892024-02-07T18:59:22.140ZMax gen: 100, Max flush: 172
110902024-02-07T18:59:22.141ZError: Difference in extent metadata found!
110912024-02-07T18:59:22.141ZA Difference in extent metadata is expected here
110922024-02-07T18:59:22.141ZOn loop 50
110932024-02-07T18:59:22.141Z
110942024-02-07T18:59:22.141ZVerifying data now
110952024-02-07T18:59:27.216ZNo change after verify
110962024-02-07T18:59:27.216ZLoop: 50 Downstairs dump after verify (and repair):
110972024-02-07T18:59:27.231ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110982024-02-07T18:59:27.231Z 0 000-019 96 96 96 161 161 161 F F F
110992024-02-07T18:59:27.231Z 1 020-039 88 88 88 145 145 145 F F F
111002024-02-07T18:59:27.232Z 2 040-059 92 92 92 151 151 151 F F F
111012024-02-07T18:59:27.232Z 3 060-079 91 91 91 150 150 150 F F F
111022024-02-07T18:59:27.232Z 4 080-099 86 86 86 141 141 141 F F F
111032024-02-07T18:59:27.248Z 5 100-119 101 101 101 173 173 173 F F F
111042024-02-07T18:59:27.248Z 6 120-139 101 101 101 173 173 173 F F F
111052024-02-07T18:59:27.248Z 7 140-159 100 100 100 172 172 172 F F F
111062024-02-07T18:59:27.248Z 8 160-179 99 99 99 165 165 165 F F F
111072024-02-07T18:59:27.248Z 9 180-199 97 97 97 162 162 162 F F F
111082024-02-07T18:59:27.249Z 10 200-219 100 100 100 167 167 167 F F F
111092024-02-07T18:59:27.249Z 11 220-239 97 97 97 162 162 162 F F F
111102024-02-07T18:59:27.249Z 12 240-259 101 101 101 173 173 173 F F F
111112024-02-07T18:59:27.249Z 13 260-279 100 100 100 171 171 171 F F F
111122024-02-07T18:59:27.249Z 14 280-299 90 90 90 148 148 148 F F F
111132024-02-07T18:59:27.249Z 15 300-319 96 96 96 161 161 161 F F F
111142024-02-07T18:59:27.249Z 16 320-339 98 98 98 164 164 164 F F F
111152024-02-07T18:59:27.249Z 17 340-359 95 95 95 158 158 158 F F F
111162024-02-07T18:59:27.249Z 18 360-379 99 99 99 165 165 165 F F F
111172024-02-07T18:59:27.249Z 19 380-399 100 100 100 172 172 172 F F F
111182024-02-07T18:59:27.249Z 20 400-419 99 99 99 165 165 165 F F F
111192024-02-07T18:59:27.249Z 21 420-439 92 92 92 151 151 151 F F F
111202024-02-07T18:59:27.249Z 22 440-459 99 99 99 165 165 165 F F F
111212024-02-07T18:59:27.250Z 23 460-479 99 99 99 165 165 165 F F F
111222024-02-07T18:59:27.250Z 24 480-499 96 96 96 161 161 161 F F F
111232024-02-07T18:59:27.250Z 25 500-519 100 100 100 169 169 169 F F F
111242024-02-07T18:59:27.250Z 26 520-539 94 94 94 157 157 157 F F F
111252024-02-07T18:59:27.250Z 27 540-559 100 100 100 167 167 167 F F F
111262024-02-07T18:59:27.250Z 28 560-579 100 100 100 169 169 169 F F F
111272024-02-07T18:59:27.250Z 29 580-599 97 97 97 162 162 162 F F F
111282024-02-07T18:59:27.250ZMax gen: 101, Max flush: 173
111292024-02-07T18:59:27.250Z
111302024-02-07T18:59:27.250ZBegin loop 51
111312024-02-07T18:59:27.251ZDownstairs to restart: 0
111322024-02-07T18:59:27.259Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.482889005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543}
111332024-02-07T18:59:27.260Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483138941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543}
111342024-02-07T18:59:27.260Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483158288Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543}
111352024-02-07T18:59:27.260Z{"msg":"Crucible 81bab5a4-4c0a-4d30-abef-bf3ad86aaff4 has session id: e01c1305-a3f7-41f2-9da6-83dad80bf815","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483192093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111362024-02-07T18:59:27.260Z{"msg":"Upstairs opts: Upstairs UUID: 81bab5a4-4c0a-4d30-abef-bf3ad86aaff4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":Crucible runtime is spawned
111372024-02-07T18:59:27.260Z0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483241757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111382024-02-07T18:59:27.260Z{"msg":"Crucible stats registered with UUID: 81bab5a4-4c0a-4d30-abef-bf3ad86aaff4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483267733Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111392024-02-07T18:59:27.260Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483321826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543}
111402024-02-07T18:59:27.260Z{"msg":"81bab5a4-4c0a-4d30-abef-bf3ad86aaff4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483498942Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111412024-02-07T18:59:27.260Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483527328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111422024-02-07T18:59:27.260Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483548895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111432024-02-07T18:59:27.260Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483575741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111442024-02-07T18:59:27.260Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483603658Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111452024-02-07T18:59:27.261Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483651001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111462024-02-07T18:59:27.261Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483707394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111472024-02-07T18:59:27.261Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483847835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111482024-02-07T18:59:27.261Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483903387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111492024-02-07T18:59:27.261Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483931453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111502024-02-07T18:59:27.261Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.483964859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111512024-02-07T18:59:27.261Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:26.484047258Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111522024-02-07T18:59:27.261Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:26.484094102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111532024-02-07T18:59:27.261Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484129107Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111542024-02-07T18:59:27.261Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:26.484157743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111552024-02-07T18:59:27.261Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484196138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111562024-02-07T18:59:27.261Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484219014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111572024-02-07T18:59:27.262Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484241871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111582024-02-07T18:59:27.262Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484482699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111592024-02-07T18:59:27.262Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484528223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111602024-02-07T18:59:27.262Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.484662035Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111612024-02-07T18:59:27.262Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.48469978Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111622024-02-07T18:59:27.262Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485160107Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111632024-02-07T18:59:27.262Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485298679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111642024-02-07T18:59:27.268Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485723631Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111652024-02-07T18:59:27.268Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485783883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111662024-02-07T18:59:27.268Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485863922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111672024-02-07T18:59:27.268Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485907046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111682024-02-07T18:59:27.268Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485925304Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111692024-02-07T18:59:27.268Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:26.485947991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111702024-02-07T18:59:28.257Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.484582005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111712024-02-07T18:59:28.273Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.484849588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111722024-02-07T18:59:28.273Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.485914544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111732024-02-07T18:59:28.273Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.485952849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111742024-02-07T18:59:28.273Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.486501505Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111752024-02-07T18:59:28.273Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487102813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111762024-02-07T18:59:28.274Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.48713071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111772024-02-07T18:59:28.274Z{"msg":"[0]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 173, 173, 172, 165, 162, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487159936Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111782024-02-07T18:59:28.274Z{"msg":"[0]R generation[0..12]: [96, 88, 92, 91, 86, 101, 101, 100, 99, 97, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487188842Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111792024-02-07T18:59:28.274Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.48720431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111802024-02-07T18:59:28.274Z{"msg":"[1]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 173, 173, 172, 165, 162, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487219798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111812024-02-07T18:59:28.274Z{"msg":"[1]R generation[0..12]: [96, 88, 92, 91, 86, 101, 101, 100, 99, 97, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487241605Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111822024-02-07T18:59:28.275Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487256553Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111832024-02-07T18:59:28.275Z{"msg":"[2]R flush_numbers[0..12]: [161, 145, 151, 150, 141, 173, 173, 172, 165, 162, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487271341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111842024-02-07T18:59:28.275Z{"msg":"[2]R generation[0..12]: [96, 88, 92, 91, 86, 101, 101, 100, 99, 97, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487285469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111852024-02-07T18:59:28.275Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487300237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111862024-02-07T18:59:28.275ZWait for a query_work_queue command to finish before sending IO
111872024-02-07T18:59:28.275Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487327083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111882024-02-07T18:59:28.276Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487341941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111892024-02-07T18:59:28.276Z{"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487355889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111902024-02-07T18:59:28.276Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487370137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111912024-02-07T18:59:28.276Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487383795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111922024-02-07T18:59:28.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487396994Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111932024-02-07T18:59:28.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487412342Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111942024-02-07T18:59:28.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.4874259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111952024-02-07T18:59:28.277Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487439468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111962024-02-07T18:59:28.277Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487452276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111972024-02-07T18:59:28.277Z{"msg":"81bab5a4-4c0a-4d30-abef-bf3ad86aaff4 is now active with session: e01c1305-a3f7-41f2-9da6-83dad80bf815","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487465304Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111982024-02-07T18:59:28.277Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487477143Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543,"session_id":"e01c1305-a3f7-41f2-9da6-83dad80bf815"}
111992024-02-07T18:59:28.277Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:27.487489441Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543}
112002024-02-07T18:59:28.277ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112012024-02-07T18:59:28.277ZLoading write count information from file "/tmp/test_repair_verify.data"
112022024-02-07T18:59:28.277ZRun Repair workload
112032024-02-07T18:59:28.277Z01/30 Write block 477 len 36864 data: 7 7 7 7 6 6 8 7 5
112042024-02-07T18:59:28.278Z02/30 Flush
112052024-02-07T18:59:28.278Z03/30 Write block 39 len 28672 data: 4 6 6 6 6 7 6
112062024-02-07T18:59:28.278Z04/30 Read block 519 len 4096
112072024-02-07T18:59:28.287Z05/30 Read block 460 len 28672
112082024-02-07T18:59:28.287Z06/30 Write block 388 len 12288 data: 11 11 6
112092024-02-07T18:59:28.287Z07/30 Read block 594 len 20480
112102024-02-07T18:59:28.287Z08/30 Write block 421 len 20480 data: 5 4 4 6 11
112112024-02-07T18:59:28.287Z09/30 Read block 456 len 36864
112122024-02-07T18:59:28.288Z10/30 Read block 146 len 16384
112132024-02-07T18:59:28.288Z11/30 Write block 266 len 16384 data: 9 9 8 8
112142024-02-07T18:59:28.288Z12/30 Read block 484 len 20480
112152024-02-07T18:59:28.288Z13/30 Read block 244 len 12288
112162024-02-07T18:59:28.288Z14/30 Read block 86 len 40960
112172024-02-07T18:59:28.289Z15/30 Write block 266 len 24576 data: 10 10 9 9 6 7
112182024-02-07T18:59:28.289Z16/30 Write block 549 len 8192 data: 11 10
112192024-02-07T18:59:28.289Z17/30 Write block 452 len 40960 data: 7 9 8 7 6 11 13 12 11 10
112202024-02-07T18:59:28.289Z18/30 Write block 27 len 28672 data: 5 8 10 11 12 10 10
112212024-02-07T18:59:28.290Z19/30 Read block 80 len 28672
112222024-02-07T18:59:28.290Z20/30 Flush
112232024-02-07T18:59:28.290Z21/30 Write block 568 len 36864 data: 10 10 10 9 6 6 5 6 5
112242024-02-07T18:59:28.290Z22/30 Write block 418 len 24576 data: 9 6 6 6 5 5
112252024-02-07T18:59:28.291Z23/30 Flush
112262024-02-07T18:59:28.291Z24/30 Write block 126 len 4096 data: 7
112272024-02-07T18:59:28.291Z25/30 Read block 548 len 12288
112282024-02-07T18:59:28.291Z26/30 Write block 354 len 4096 data: 9
112292024-02-07T18:59:28.301Z27/30 Read block 177 len 20480
112302024-02-07T18:59:28.302Z28/30 Read block 252 len 8192
112312024-02-07T18:59:28.302Z29/30 Write block 554 len 36864 data: 6 8 7 5 7 9 10 8 8
112322024-02-07T18:59:28.302Z30/30 Read block 402 len 40960
112332024-02-07T18:59:28.302Z----------------------------------------------------------------
112342024-02-07T18:59:28.302Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
112352024-02-07T18:59:28.303ZStates: Active Active Active
112362024-02-07T18:59:28.303ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112372024-02-07T18:59:28.303Z 1 Acked 1000 Write 9 Sent Done Done false
112382024-02-07T18:59:28.303Z 2 Acked 1001 Flush 0 Sent Done Done false
112392024-02-07T18:59:28.303Z 3 Acked 1002 Write 7 Sent Done Done false
112402024-02-07T18:59:28.304Z 4 Acked 1003 Read 1 Sent Done Done false
112412024-02-07T18:59:28.304Z 5 Acked 1004 Read 7 Sent Done Done false
112422024-02-07T18:59:28.304Z 6 Acked 1005 Write 3 Sent Done Done false
112432024-02-07T18:59:28.304Z 7 Acked 1006 Read 5 Sent Done Done false
112442024-02-07T18:59:28.304Z 8 Acked 1007 Write 5 Sent Done Done false
112452024-02-07T18:59:28.304Z 9 Acked 1008 Read 9 Sent Done Done false
112462024-02-07T18:59:28.305Z 10 Acked 1009 Read 4 Sent Done Done false
112472024-02-07T18:59:28.305Z 11 Acked 1010 Write 4 Sent Done Done false
112482024-02-07T18:59:28.305Z 12 Acked 1011 Read 5 Sent Done Done false
112492024-02-07T18:59:28.305Z 13 Acked 1012 Read 3 Sent Done Done false
112502024-02-07T18:59:28.305Z 14 Acked 1013 Read 10 Sent Done Done false
112512024-02-07T18:59:28.305Z 15 Acked 1014 Write 6 Sent Done Done false
112522024-02-07T18:59:28.305Z 16 Acked 1015 Write 2 Sent Done Done false
112532024-02-07T18:59:28.306Z 17 Acked 1016 Write 10 Sent Done Done false
112542024-02-07T18:59:28.306Z 18 Acked 1017 Write 7 Sent Done Done false
112552024-02-07T18:59:28.306Z 19 Acked 1018 Read 7 Sent Done Done false
112562024-02-07T18:59:28.306Z 20 Acked 1019 Flush 0 Sent Done Done false
112572024-02-07T18:59:28.306Z 21 Acked 1020 Write 9 Sent Done Done false
112582024-02-07T18:59:28.306Z 22 Acked 1021 Write 6 Sent Done Done false
112592024-02-07T18:59:28.306Z 23 Acked 1022 Flush 0 Sent Done Done false
112602024-02-07T18:59:28.307Z 24 Acked 1023 Write 1 Sent Done Done false
112612024-02-07T18:59:28.307Z 25 Acked 1024 Read 3 Sent Done Done false
112622024-02-07T18:59:28.307Z 26 Acked 1025 Write 1 Sent Done Done false
112632024-02-07T18:59:28.307Z 27 Acked 1026 Read 5 Sent Done Done false
112642024-02-07T18:59:28.307Z 28 Acked 1027 Read 2 Sent Done Done false
112652024-02-07T18:59:28.308Z 29 Acked 1028 Write 9 Sent Done Done false
112662024-02-07T18:59:28.308Z 30 Acked 1029 Read 10 Sent Done Done false
112672024-02-07T18:59:28.308Z STATES DS:0 DS:1 DS:2 TOTAL
112682024-02-07T18:59:28.308Z New 0 0 0 0
112692024-02-07T18:59:28.308Z Sent 30 0 0 30
112702024-02-07T18:59:28.309Z Done 0 30 30 60
112712024-02-07T18:59:28.309Z Skipped 0 0 0 0
112722024-02-07T18:59:28.309Z Error 0 0 0 0
112732024-02-07T18:59:28.309ZLast Flush: 0 1022 1022
112742024-02-07T18:59:28.309ZDownstairs last five completed:
112752024-02-07T18:59:28.309ZUpstairs last five completed: 30 29 28 27 26
112762024-02-07T18:59:28.309Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:27.520274587Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1543}
112772024-02-07T18:59:28.310ZWrote out file "/tmp/test_repair_verify.data"
112782024-02-07T18:59:28.310Z
112792024-02-07T18:59:28.324ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112802024-02-07T18:59:28.324ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112812024-02-07T18:59:28.324Z 0 000-019 96 96 96 161 161 161 F F F
112822024-02-07T18:59:28.324Z 1 020-039 88 102 102 145 175 175 F F F <---
112832024-02-07T18:59:28.324Z 2 040-059 92 102 102 151 175 175 F F F <---
112842024-02-07T18:59:28.325Z 3 060-079 91 91 91 150 150 150 F F F
112852024-02-07T18:59:28.325Z 4 080-099 86 86 86 141 141 141 F F F
112862024-02-07T18:59:28.325Z 5 100-119 101 101 101 173 173 173 F F F
112872024-02-07T18:59:28.325Z 6 120-139 101 101 101 173 173 173 F T T <---
112882024-02-07T18:59:28.325Z 7 140-159 100 100 100 172 172 172 F F F
112892024-02-07T18:59:28.325Z 8 160-179 99 99 99 165 165 165 F F F
112902024-02-07T18:59:28.325Z 9 180-199 97 97 97 162 162 162 F F F
112912024-02-07T18:59:28.326Z 10 200-219 100 100 100 167 167 167 F F F
112922024-02-07T18:59:28.326Z 11 220-239 97 97 97 162 162 162 F F F
112932024-02-07T18:59:28.326Z 12 240-259 101 101 101 173 173 173 F F F
112942024-02-07T18:59:28.326Z 13 260-279 100 102 102 171 175 175 F F F <---
112952024-02-07T18:59:28.326Z 14 280-299 90 90 90 148 148 148 F F F
112962024-02-07T18:59:28.327Z 15 300-319 96 96 96 161 161 161 F F F
112972024-02-07T18:59:28.327Z 16 320-339 98 98 98 164 164 164 F F F
112982024-02-07T18:59:28.327Z 17 340-359 95 95 95 158 158 158 F T T <---
112992024-02-07T18:59:28.328Z 18 360-379 99 99 99 165 165 165 F F F
113002024-02-07T18:59:28.328Z 19 380-399 100 102 102 172 175 175 F F F <---
113012024-02-07T18:59:28.328Z 20 400-419 99 102 102 165 176 176 F F F <---
113022024-02-07T18:59:28.328Z 21 420-439 92 102 102 151 176 176 F F F <---
113032024-02-07T18:59:28.328Z 22 440-459 99 102 102 165 175 175 F F F <---
113042024-02-07T18:59:28.328Z 23 460-479 99 102 102 165 175 175 F F F <---
113052024-02-07T18:59:28.328Z 24 480-499 96 102 102 161 174 174 F F F <---
113062024-02-07T18:59:28.329Z 25 500-519 100 100 100 169 169 169 F F F
113072024-02-07T18:59:28.329Z 26 520-539 94 94 94 157 157 157 F F F
113082024-02-07T18:59:28.329Z 27 540-559 100 102 102 167 175 175 F T T <---
113092024-02-07T18:59:28.329Z 28 560-579 100 102 102 169 176 176 F T T <---
113102024-02-07T18:59:28.329Z 29 580-599 97 97 97 162 162 162 F F F
113112024-02-07T18:59:28.329ZMax gen: 102, Max flush: 176
113122024-02-07T18:59:28.330ZError: Difference in extent metadata found!
113132024-02-07T18:59:28.330ZA Difference in extent metadata is expected here
113142024-02-07T18:59:28.330ZOn loop 51
113152024-02-07T18:59:28.330Z
113162024-02-07T18:59:28.341ZVerifying data now
113172024-02-07T18:59:32.955ZNo change after verify
113182024-02-07T18:59:32.955ZLoop: 51 Downstairs dump after verify (and repair):
113192024-02-07T18:59:32.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113202024-02-07T18:59:32.971Z 0 000-019 96 96 96 161 161 161 F F F
113212024-02-07T18:59:32.971Z 1 020-039 102 102 102 175 175 175 F F F
113222024-02-07T18:59:32.971Z 2 040-059 102 102 102 175 175 175 F F F
113232024-02-07T18:59:32.971Z 3 060-079 91 91 91 150 150 150 F F F
113242024-02-07T18:59:32.971Z 4 080-099 86 86 86 141 141 141 F F F
113252024-02-07T18:59:32.971Z 5 100-119 101 101 101 173 173 173 F F F
113262024-02-07T18:59:32.971Z 6 120-139 103 103 103 177 177 177 F F F
113272024-02-07T18:59:32.971Z 7 140-159 100 100 100 172 172 172 F F F
113282024-02-07T18:59:32.971Z 8 160-179 99 99 99 165 165 165 F F F
113292024-02-07T18:59:32.971Z 9 180-199 97 97 97 162 162 162 F F F
113302024-02-07T18:59:32.971Z 10 200-219 100 100 100 167 167 167 F F F
113312024-02-07T18:59:32.984Z 11 220-239 97 97 97 162 162 162 F F F
113322024-02-07T18:59:32.984Z 12 240-259 101 101 101 173 173 173 F F F
113332024-02-07T18:59:32.985Z 13 260-279 102 102 102 175 175 175 F F F
113342024-02-07T18:59:32.985Z 14 280-299 90 90 90 148 148 148 F F F
113352024-02-07T18:59:32.985Z 15 300-319 96 96 96 161 161 161 F F F
113362024-02-07T18:59:32.985Z 16 320-339 98 98 98 164 164 164 F F F
113372024-02-07T18:59:32.985Z 17 340-359 103 103 103 177 177 177 F F F
113382024-02-07T18:59:32.985Z 18 360-379 99 99 99 165 165 165 F F F
113392024-02-07T18:59:32.985Z 19 380-399 102 102 102 175 175 175 F F F
113402024-02-07T18:59:32.985Z 20 400-419 102 102 102 176 176 176 F F F
113412024-02-07T18:59:32.985Z 21 420-439 102 102 102 176 176 176 F F F
113422024-02-07T18:59:32.985Z 22 440-459 102 102 102 175 175 175 F F F
113432024-02-07T18:59:32.985Z 23 460-479 102 102 102 175 175 175 F F F
113442024-02-07T18:59:32.985Z 24 480-499 102 102 102 174 174 174 F F F
113452024-02-07T18:59:32.986Z 25 500-519 100 100 100 169 169 169 F F F
113462024-02-07T18:59:32.986Z 26 520-539 94 94 94 157 157 157 F F F
113472024-02-07T18:59:32.986Z 27 540-559 103 103 103 177 177 177 F F F
113482024-02-07T18:59:32.986Z 28 560-579 103 103 103 177 177 177 F F F
113492024-02-07T18:59:32.986Z 29 580-599 97 97 97 162 162 162 F F F
113502024-02-07T18:59:32.986ZMax gen: 103, Max flush: 177
113512024-02-07T18:59:32.986Z
113522024-02-07T18:59:32.986ZBegin loop 52
113532024-02-07T18:59:32.986ZDownstairs to restart: 0
113542024-02-07T18:59:33.006Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221188846Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551}
113552024-02-07T18:59:33.006Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221426404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551}
113562024-02-07T18:59:33.006Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.22146152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551}
113572024-02-07T18:59:33.006Z{"msg":"Crucible 186e9fc4-e82c-49fc-ae7c-a1a9f7fad898 has session id: 8c7353f9-6a9f-4b61-b6ea-36388524f5db","v":0,"name":"Crucible runtime is spawned
113582024-02-07T18:59:33.006Zcrucible","level":30,"time":"2024-02-07T18:59:32.221549168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113592024-02-07T18:59:33.006Z{"msg":"Upstairs opts: Upstairs UUID: 186e9fc4-e82c-49fc-ae7c-a1a9f7fad898, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.22160385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113602024-02-07T18:59:33.006Z{"msg":"Crucible stats registered with UUID: 186e9fc4-e82c-49fc-ae7c-a1a9f7fad898","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221639765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113612024-02-07T18:59:33.007Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221655683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551}
113622024-02-07T18:59:33.007Z{"msg":"186e9fc4-e82c-49fc-ae7c-a1a9f7fad898 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221810582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113632024-02-07T18:59:33.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221837439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113642024-02-07T18:59:33.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221852596Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113652024-02-07T18:59:33.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221890182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113662024-02-07T18:59:33.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221917418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113672024-02-07T18:59:33.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221949743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113682024-02-07T18:59:33.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.221981499Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113692024-02-07T18:59:33.007Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222006306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113702024-02-07T18:59:33.007Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222031112Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113712024-02-07T18:59:33.007Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222054719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113722024-02-07T18:59:33.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222204149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113732024-02-07T18:59:33.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222376906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113742024-02-07T18:59:33.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222410981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113752024-02-07T18:59:33.007Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222823105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113762024-02-07T18:59:33.008Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222856131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113772024-02-07T18:59:33.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.222964436Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113782024-02-07T18:59:33.008Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.22300686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113792024-02-07T18:59:33.008Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223317329Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113802024-02-07T18:59:33.008Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.2234533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113812024-02-07T18:59:33.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223829679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113822024-02-07T18:59:33.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223868664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113832024-02-07T18:59:33.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223884912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113842024-02-07T18:59:33.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223951053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113852024-02-07T18:59:33.008Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223977599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113862024-02-07T18:59:33.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.223992947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113872024-02-07T18:59:33.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.224053889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113882024-02-07T18:59:33.008Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.224079485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113892024-02-07T18:59:33.021Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.224620142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113902024-02-07T18:59:33.021Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225135932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113912024-02-07T18:59:33.021Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225178577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113922024-02-07T18:59:33.021Z{"msg":"[0]R flush_numbers[0..12]: [161, 175, 175, 150, 141, 173, 177, 172, 165, 162, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225199354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113932024-02-07T18:59:33.021Z{"msg":"[0]R generation[0..12]: [96, 102, 102, 91, 86, 101, 103, 100, 99, 97, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.22522708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113942024-02-07T18:59:33.022Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225245698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113952024-02-07T18:59:33.022Z{"msg":"[1]R flush_numbers[0..12]: [161, 175, 175, 150, 141, 173, 177, 172, 165, 162, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225272914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113962024-02-07T18:59:33.022Z{"msg":"[1]R generation[0..12]: [96, 102, 102, 91, 86, 101, 103, 100, 99, 97, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225290722Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113972024-02-07T18:59:33.022Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225308939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
113982024-02-07T18:59:33.022ZWait for a query_work_queue command to finish before sending IO
113992024-02-07T18:59:33.022Z{"msg":"[2]R flush_numbers[0..12]: [161, 175, 175, 150, 141, 173, 177, 172, 165, 162, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225340135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114002024-02-07T18:59:33.022Z{"msg":"[2]R generation[0..12]: [96, 102, 102, 91, 86, 101, 103, 100, 99, 97, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225367391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114012024-02-07T18:59:33.023Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225385729Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114022024-02-07T18:59:33.023Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225411335Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114032024-02-07T18:59:33.023Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225428773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114042024-02-07T18:59:33.023Z{"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225454049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114052024-02-07T18:59:33.023Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225471037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114062024-02-07T18:59:33.023Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225503603Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114072024-02-07T18:59:33.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225519271Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114082024-02-07T18:59:33.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225535758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114092024-02-07T18:59:33.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225561875Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114102024-02-07T18:59:33.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225579612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114112024-02-07T18:59:33.023Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225603539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114122024-02-07T18:59:33.024Z{"msg":"186e9fc4-e82c-49fc-ae7c-a1a9f7fad898 is now active with session: 8c7353f9-6a9f-4b61-b6ea-36388524f5db","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225619987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114132024-02-07T18:59:33.024Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225643324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551,"session_id":"8c7353f9-6a9f-4b61-b6ea-36388524f5db"}
114142024-02-07T18:59:33.024Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:32.225659312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551}
114152024-02-07T18:59:33.024ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114162024-02-07T18:59:33.024ZLoading write count information from file "/tmp/test_repair_verify.data"
114172024-02-07T18:59:33.024ZRun Repair workload
114182024-02-07T18:59:33.024Z01/30 Write block 338 len 40960 data: 9 11 9 6 6 6 7 10 11 11
114192024-02-07T18:59:33.024Z02/30 Read block 274 len 4096
114202024-02-07T18:59:33.024Z03/30 Write block 55 len 8192 data: 12 12
114212024-02-07T18:59:33.025Z04/30 Read block 498 len 12288
114222024-02-07T18:59:33.025Z05/30 Write block 405 len 36864 data: 5 7 7 7 9 12 12 9 12
114232024-02-07T18:59:33.025Z06/30 Flush
114242024-02-07T18:59:33.025Z07/30 Write block 582 len 16384 data: 9 7 7 8
114252024-02-07T18:59:33.025Z08/30 Flush
114262024-02-07T18:59:33.025Z09/30 Write block 242 len 4096 data: 20
114272024-02-07T18:59:33.025Z10/30 Write block 482 len 24576 data: 7 9 8 6 5 6
114282024-02-07T18:59:33.025Z11/30 Read block 435 len 32768
114292024-02-07T18:59:33.025Z12/30 Read block 230 len 40960
114302024-02-07T18:59:33.026Z13/30 Write block 397 len 28672 data: 7 7 7 9 7 9 6
114312024-02-07T18:59:33.026Z14/30 Flush
114322024-02-07T18:59:33.038Z15/30 Flush
114332024-02-07T18:59:33.039Z16/30 Write block 573 len 20480 data: 7 6 7 6 5
114342024-02-07T18:59:33.039Z17/30 Flush
114352024-02-07T18:59:33.039Z18/30 Write block 284 len 36864 data: 9 9 11 12 12 12 13 11 9
114362024-02-07T18:59:33.040Z19/30 Read block 174 len 20480
114372024-02-07T18:59:33.040Z20/30 Read block 147 len 16384
114382024-02-07T18:59:33.040Z21/30 Read block 346 len 28672
114392024-02-07T18:59:33.040Z22/30 Read block 236 len 20480
114402024-02-07T18:59:33.040Z23/30 Write block 286 len 28672 data: 12 13 13 13 14 12 10
114412024-02-07T18:59:33.040Z24/30 Write block 14 len 8192 data: 9 7
114422024-02-07T18:59:33.040Z25/30 Read block 316 len 4096
114432024-02-07T18:59:33.041Z26/30 Flush
114442024-02-07T18:59:33.041Z27/30 Write block 399 len 12288 data: 8 10 8
114452024-02-07T18:59:33.041Z28/30 Write block 188 len 8192 data: 5 5
114462024-02-07T18:59:33.041Z29/30 Write block 399 len 32768 data: 9 11 9 10 7 6 6 8
114472024-02-07T18:59:33.041Z30/30 Read block 78 len 40960
114482024-02-07T18:59:33.042Z----------------------------------------------------------------
114492024-02-07T18:59:33.042Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
114502024-02-07T18:59:33.042ZStates: Active Active Active
114512024-02-07T18:59:33.042ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114522024-02-07T18:59:33.042Z 1 Acked 1000 Write 10 Done Done Done false
114532024-02-07T18:59:33.043Z 2 Acked 1001 Read 1 Done Done Done false
114542024-02-07T18:59:33.043Z 3 Acked 1002 Write 2 Done Done Done false
114552024-02-07T18:59:33.043Z 4 Acked 1003 Read 3 Done Done Done false
114562024-02-07T18:59:33.043Z 5 Acked 1004 Write 9 Sent Done Done false
114572024-02-07T18:59:33.043Z 6 Acked 1005 Flush 0 Sent Done Done false
114582024-02-07T18:59:33.043Z 7 Acked 1006 Write 4 Sent Done Done false
114592024-02-07T18:59:33.044Z 8 Acked 1007 Flush 0 Sent Done Done false
114602024-02-07T18:59:33.044Z 9 Acked 1008 Write 1 Sent Done Done false
114612024-02-07T18:59:33.044Z 10 Acked 1009 Write 6 Sent Done Done false
114622024-02-07T18:59:33.044Z 11 Acked 1010 Read 8 Sent Done Done false
114632024-02-07T18:59:33.044Z 12 Acked 1011 Read 10 Sent Done Done false
114642024-02-07T18:59:33.045Z 13 Acked 1012 Write 7 Sent Done Done false
114652024-02-07T18:59:33.045Z 14 Acked 1013 Flush 0 Sent Done Done false
114662024-02-07T18:59:33.045Z 15 Acked 1014 Flush 0 Sent Done Done false
114672024-02-07T18:59:33.045Z 16 Acked 1015 Write 5 Sent Done Done false
114682024-02-07T18:59:33.045Z 17 Acked 1016 Flush 0 Sent Done Done false
114692024-02-07T18:59:33.045Z 18 Acked 1017 Write 9 Sent Done Done false
114702024-02-07T18:59:33.046Z 19 Acked 1018 Read 5 Sent Done Done false
114712024-02-07T18:59:33.046Z 20 Acked 1019 Read 4 Sent Done Done false
114722024-02-07T18:59:33.046Z 21 Acked 1020 Read 7 Sent Done Done false
114732024-02-07T18:59:33.046Z 22 Acked 1021 Read 5 Sent Done Done false
114742024-02-07T18:59:33.046Z 23 Acked 1022 Write 7 Sent Done Done false
114752024-02-07T18:59:33.046Z 24 Acked 1023 Write 2 Sent Done Done false
114762024-02-07T18:59:33.046Z 25 Acked 1024 Read 1 Sent Done Done false
114772024-02-07T18:59:33.046Z 26 Acked 1025 Flush 0 Sent Done Done false
114782024-02-07T18:59:33.047Z 27 Acked 1026 Write 3 Sent Done Done false
114792024-02-07T18:59:33.047Z 28 Acked 1027 Write 2 Sent Done Done false
114802024-02-07T18:59:33.048Z 29 Acked 1028 Write 8 Sent Done Done false
114812024-02-07T18:59:33.048Z 30 Acked 1029 Read 10 Sent Sent Done false
114822024-02-07T18:59:33.048Z STATES DS:0 DS:1 DS:2 TOTAL
114832024-02-07T18:59:33.048Z New 0 0 0 0
114842024-02-07T18:59:33.048Z Sent 26 1 0 27
114852024-02-07T18:59:33.049Z Done 4 29 30 63
114862024-02-07T18:59:33.049Z Skipped 0 0 0 0
114872024-02-07T18:59:33.049Z Error 0 0 0 0
114882024-02-07T18:59:33.049ZLast Flush: 0 1025 1025
114892024-02-07T18:59:33.049ZDownstairs last five completed:
114902024-02-07T18:59:33.049ZUpstairs last five completed: 30 29 28 27 26
114912024-02-07T18:59:33.049Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:32.259817292Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1551}
114922024-02-07T18:59:33.049ZWrote out file "/tmp/test_repair_verify.data"
114932024-02-07T18:59:33.055Z
114942024-02-07T18:59:33.055ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114952024-02-07T18:59:33.055ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114962024-02-07T18:59:33.055Z 0 000-019 96 104 104 161 183 183 F F F <---
114972024-02-07T18:59:33.055Z 1 020-039 102 102 102 175 175 175 F F F
114982024-02-07T18:59:33.055Z 2 040-059 102 104 104 175 178 178 T F F <---
114992024-02-07T18:59:33.055Z 3 060-079 91 91 91 150 150 150 F F F
115002024-02-07T18:59:33.055Z 4 080-099 86 86 86 141 141 141 F F F
115012024-02-07T18:59:33.055Z 5 100-119 101 101 101 173 173 173 F F F
115022024-02-07T18:59:33.055Z 6 120-139 103 103 103 177 177 177 F F F
115032024-02-07T18:59:33.055Z 7 140-159 100 100 100 172 172 172 F F F
115042024-02-07T18:59:33.055Z 8 160-179 99 99 99 165 165 165 F F F
115052024-02-07T18:59:33.055Z 9 180-199 97 97 97 162 162 162 F T T <---
115062024-02-07T18:59:33.055Z 10 200-219 100 100 100 167 167 167 F F F
115072024-02-07T18:59:33.055Z 11 220-239 97 97 97 162 162 162 F F F
115082024-02-07T18:59:33.055Z 12 240-259 101 104 104 173 180 180 F F F <---
115092024-02-07T18:59:33.056Z 13 260-279 102 102 102 175 175 175 F F F
115102024-02-07T18:59:33.056Z 14 280-299 90 104 104 148 183 183 F F F <---
115112024-02-07T18:59:33.056Z 15 300-319 96 96 96 161 161 161 F F F
115122024-02-07T18:59:33.056Z 16 320-339 98 104 104 164 178 178 T F F <---
115132024-02-07T18:59:33.056Z 17 340-359 103 104 104 177 178 178 T F F <---
115142024-02-07T18:59:33.056Z 18 360-379 99 99 99 165 165 165 F F F
115152024-02-07T18:59:33.056Z 19 380-399 102 104 104 175 180 180 F T T <---
115162024-02-07T18:59:33.056Z 20 400-419 102 104 104 176 180 180 F T T <---
115172024-02-07T18:59:33.056Z 21 420-439 102 102 102 176 176 176 F F F
115182024-02-07T18:59:33.056Z 22 440-459 102 102 102 175 175 175 F F F
115192024-02-07T18:59:33.056Z 23 460-479 102 102 102 175 175 175 F F F
115202024-02-07T18:59:33.056Z 24 480-499 102 104 104 174 180 180 F F F <---
115212024-02-07T18:59:33.056Z 25 500-519 100 100 100 169 169 169 F F F
115222024-02-07T18:59:33.056Z 26 520-539 94 94 94 157 157 157 F F F
115232024-02-07T18:59:33.056Z 27 540-559 103 103 103 177 177 177 F F F
115242024-02-07T18:59:33.056Z 28 560-579 103 104 104 177 182 182 F F F <---
115252024-02-07T18:59:33.056Z 29 580-599 97 104 104 162 179 179 F F F <---
115262024-02-07T18:59:33.056ZMax gen: 104, Max flush: 183
115272024-02-07T18:59:33.060ZError: Difference in extent metadata found!
115282024-02-07T18:59:33.060ZA Difference in extent metadata is expected here
115292024-02-07T18:59:33.060ZOn loop 52
115302024-02-07T18:59:33.060Z
115312024-02-07T18:59:33.063ZVerifying data now
115322024-02-07T18:59:38.651ZNo change after verify
115332024-02-07T18:59:38.652ZLoop: 52 Downstairs dump after verify (and repair):
115342024-02-07T18:59:38.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115352024-02-07T18:59:38.667Z 0 000-019 104 104 104 183 183 183 F F F
115362024-02-07T18:59:38.667Z 1 020-039 102 102 102 175 175 175 F F F
115372024-02-07T18:59:38.667Z 2 040-059 104 104 104 178 178 178 F F F
115382024-02-07T18:59:38.667Z 3 060-079 91 91 91 150 150 150 F F F
115392024-02-07T18:59:38.667Z 4 080-099 86 86 86 141 141 141 F F F
115402024-02-07T18:59:38.667Z 5 100-119 101 101 101 173 173 173 F F F
115412024-02-07T18:59:38.667Z 6 120-139 103 103 103 177 177 177 F F F
115422024-02-07T18:59:38.668Z 7 140-159 100 100 100 172 172 172 F F F
115432024-02-07T18:59:38.668Z 8 160-179 99 99 99 165 165 165 F F F
115442024-02-07T18:59:38.668Z 9 180-199 105 105 105 184 184 184 F F F
115452024-02-07T18:59:38.668Z 10 200-219 100 100 100 167 167 167 F F F
115462024-02-07T18:59:38.671Z 11 220-239 97 97 97 162 162 162 F F F
115472024-02-07T18:59:38.671Z 12 240-259 104 104 104 180 180 180 F F F
115482024-02-07T18:59:38.671Z 13 260-279 102 102 102 175 175 175 F F F
115492024-02-07T18:59:38.671Z 14 280-299 104 104 104 183 183 183 F F F
115502024-02-07T18:59:38.671Z 15 300-319 96 96 96 161 161 161 F F F
115512024-02-07T18:59:38.671Z 16 320-339 104 104 104 178 178 178 F F F
115522024-02-07T18:59:38.671Z 17 340-359 104 104 104 178 178 178 F F F
115532024-02-07T18:59:38.671Z 18 360-379 99 99 99 165 165 165 F F F
115542024-02-07T18:59:38.671Z 19 380-399 105 105 105 184 184 184 F F F
115552024-02-07T18:59:38.671Z 20 400-419 105 105 105 184 184 184 F F F
115562024-02-07T18:59:38.671Z 21 420-439 102 102 102 176 176 176 F F F
115572024-02-07T18:59:38.672Z 22 440-459 102 102 102 175 175 175 F F F
115582024-02-07T18:59:38.672Z 23 460-479 102 102 102 175 175 175 F F F
115592024-02-07T18:59:38.672Z 24 480-499 104 104 104 180 180 180 F F F
115602024-02-07T18:59:38.672Z 25 500-519 100 100 100 169 169 169 F F F
115612024-02-07T18:59:38.672Z 26 520-539 94 94 94 157 157 157 F F F
115622024-02-07T18:59:38.672Z 27 540-559 103 103 103 177 177 177 F F F
115632024-02-07T18:59:38.672Z 28 560-579 104 104 104 182 182 182 F F F
115642024-02-07T18:59:38.672Z 29 580-599 104 104 104 179 179 179 F F F
115652024-02-07T18:59:38.672ZMax gen: 105, Max flush: 184
115662024-02-07T18:59:38.675Z
115672024-02-07T18:59:38.675ZBegin loop 53
115682024-02-07T18:59:38.675ZDownstairs to restart: 1
115692024-02-07T18:59:38.692Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921043519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559}
115702024-02-07T18:59:38.692Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921285476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559}
115712024-02-07T18:59:38.692Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921304734Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559}
115722024-02-07T18:59:38.692Z{"msg":"Crucible 1e11fbf8-6fbb-4b1a-a39b-466b3e2ac462 has session id: 04652475-ac64-4570-8168-fffe9b41b738","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921320841Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115732024-02-07T18:59:38.698Z{"msg":"Upstairs opts: Upstairs UUID: 1e11fbf8-6fbb-4b1a-a39b-466b3e2ac462, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
115742024-02-07T18:59:38.699Z,"time":"2024-02-07T18:59:37.921369205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115752024-02-07T18:59:38.699Z{"msg":"Crucible stats registered with UUID: 1e11fbf8-6fbb-4b1a-a39b-466b3e2ac462","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921452164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115762024-02-07T18:59:38.699Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921501957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559}
115772024-02-07T18:59:38.699Z{"msg":"1e11fbf8-6fbb-4b1a-a39b-466b3e2ac462 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921594375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115782024-02-07T18:59:38.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921633479Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115792024-02-07T18:59:38.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921672554Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115802024-02-07T18:59:38.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921710439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115812024-02-07T18:59:38.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921750003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115822024-02-07T18:59:38.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921779099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115832024-02-07T18:59:38.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921825473Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115842024-02-07T18:59:38.699Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.92185274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115852024-02-07T18:59:38.699Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.921891624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115862024-02-07T18:59:38.699Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.92192168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115872024-02-07T18:59:38.699Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:37.921963805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115882024-02-07T18:59:38.699Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:37.92199584Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115892024-02-07T18:59:38.699Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922037615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115902024-02-07T18:59:38.699Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922064211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115912024-02-07T18:59:38.699Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:37.922101916Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115922024-02-07T18:59:38.699Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922119093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115932024-02-07T18:59:38.699Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.92214864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115942024-02-07T18:59:38.699Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922314087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115952024-02-07T18:59:38.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922801951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115962024-02-07T18:59:38.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922838286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115972024-02-07T18:59:38.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922854044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115982024-02-07T18:59:38.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.922868122Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
115992024-02-07T18:59:38.700Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.923439025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116002024-02-07T18:59:38.700Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.923563598Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116012024-02-07T18:59:38.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.924014747Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116022024-02-07T18:59:38.700Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.924080758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116032024-02-07T18:59:38.700Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.924102775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116042024-02-07T18:59:38.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.924124992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116052024-02-07T18:59:38.700Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.92413948Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116062024-02-07T18:59:38.700Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:37.924160117Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116072024-02-07T18:59:39.696Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.923885894Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116082024-02-07T18:59:39.697Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.924045192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116092024-02-07T18:59:39.726Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.92487103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116102024-02-07T18:59:39.726Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.924920354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116112024-02-07T18:59:39.727Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.925468849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116122024-02-07T18:59:39.727Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926073068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116132024-02-07T18:59:39.727Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926117422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116142024-02-07T18:59:39.727Z{"msg":"[0]R flush_numbers[0..12]: [183, 175, 178, 150, 141, 173, 177, 172, 165, 184, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926154037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116152024-02-07T18:59:39.727Z{"msg":"[0]R generation[0..12]: [104, 102, 104, 91, 86, 101, 103, 100, 99, 105, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926192671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116162024-02-07T18:59:39.727Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926217808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116172024-02-07T18:59:39.727Z{"msg":"[1]R flush_numbers[0..12]: [183, 175, 178, 150, 141, 173, 177, 172, 165, 184, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926232746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116182024-02-07T18:59:39.727Z{"msg":"[1]R generation[0..12]: [104, 102, 104, 91, 86, 101, 103, 100, 99, 105, 100, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926253263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116192024-02-07T18:59:39.727Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926268091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116202024-02-07T18:59:39.727Z{"msg":"[2]R flush_numbers[0..12]: [183, 175, 178, 150, 141, 173, 177, 172, 165, 184, 167, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926289118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116212024-02-07T18:59:39.727Z{"msg":"[2]R generation[0..12]: [104, 102, 104, 91, 86, 101, 103, 100, 99, 105, 100, 97]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
116222024-02-07T18:59:39.727Z,"time":"2024-02-07T18:59:38.926303367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116232024-02-07T18:59:39.728Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926329043Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116242024-02-07T18:59:39.728Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.92635011Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116252024-02-07T18:59:39.728Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926365028Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116262024-02-07T18:59:39.728Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926390355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116272024-02-07T18:59:39.728Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926410892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116282024-02-07T18:59:39.728Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926432519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116292024-02-07T18:59:39.728Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926445217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116302024-02-07T18:59:39.728Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926473054Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116312024-02-07T18:59:39.728Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926493301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116322024-02-07T18:59:39.728Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926516888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116332024-02-07T18:59:39.728Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926530516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116342024-02-07T18:59:39.728Z{"msg":"1e11fbf8-6fbb-4b1a-a39b-466b3e2ac462 is now active with session: 04652475-ac64-4570-8168-fffe9b41b738","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926545444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116352024-02-07T18:59:39.729Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926562192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559,"session_id":"04652475-ac64-4570-8168-fffe9b41b738"}
116362024-02-07T18:59:39.729Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:38.926582519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559}
116372024-02-07T18:59:39.729ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116382024-02-07T18:59:39.729ZLoading write count information from file "/tmp/test_repair_verify.data"
116392024-02-07T18:59:39.729ZRun Repair workload
116402024-02-07T18:59:39.729Z01/30 Write block 237 len 4096 data: 16
116412024-02-07T18:59:39.729Z02/30 Flush
116422024-02-07T18:59:39.730Z03/30 Write block 573 len 8192 data: 8 7
116432024-02-07T18:59:39.730Z04/30 Write block 472 len 24576 data: 7 7 7 8 9 8
116442024-02-07T18:59:39.730Z05/30 Read block 96 len 16384
116452024-02-07T18:59:39.730Z06/30 Read block 294 len 28672
116462024-02-07T18:59:39.730Z07/30 Write block 505 len 24576 data: 13 10 8 8 8 10
116472024-02-07T18:59:39.730Z08/30 Write block 346 len 24576 data: 12 12 10 11 11 10
116482024-02-07T18:59:39.730Z09/30 Read block 160 len 36864
116492024-02-07T18:59:39.730Z10/30 Read block 296 len 40960
116502024-02-07T18:59:39.730Z11/30 Read block 74 len 32768
116512024-02-07T18:59:39.730Z12/30 Write block 339 len 4096 data: 12
116522024-02-07T18:59:39.731Z13/30 Write block 348 len 8192 data: 11 12
116532024-02-07T18:59:39.731Z14/30 Read block 220 len 24576
116542024-02-07T18:59:39.731Z15/30 Read block 270 len 8192
116552024-02-07T18:59:39.731Z16/30 Write block 467 len 8192 data: 5 4
116562024-02-07T18:59:39.731Z17/30 Read block 122 len 40960
116572024-02-07T18:59:39.731Z18/30 Read block 205 len 24576
116582024-02-07T18:59:39.731Z19/30 Flush
116592024-02-07T18:59:39.771Z20/30 Write block 95 len 8192 data: 11 11
116602024-02-07T18:59:39.771Z21/30 Read block 300 len 4096
116612024-02-07T18:59:39.771Z22/30 Read block 585 len 28672
116622024-02-07T18:59:39.771Z23/30 Write block 78 len 20480 data: 9 7 6 4 5
116632024-02-07T18:59:39.771Z24/30 Write block 208 len 28672 data: 11 9 8 9 9 9 7
116642024-02-07T18:59:39.772Z25/30 Write block 546 len 16384 data: 5 9 10 12
116652024-02-07T18:59:39.772Z26/30 Flush
116662024-02-07T18:59:39.772Z27/30 Write block 281 len 32768 data: 6 7 6 10 10 13 14 14
116672024-02-07T18:59:39.772Z28/30 Write block 588 len 24576 data: 9 9 9 10 9 8
116682024-02-07T18:59:39.772Z29/30 Write block 283 len 36864 data: 7 11 11 14 15 15 14 15 13
116692024-02-07T18:59:39.772Z30/30 Write block 308 len 16384 data: 6 7 8 8
116702024-02-07T18:59:39.772Z----------------------------------------------------------------
116712024-02-07T18:59:39.772Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:28
116722024-02-07T18:59:39.772ZStates: Active Active Active
116732024-02-07T18:59:39.772ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116742024-02-07T18:59:39.773Z 3 Acked 1002 Write 2 Done Done Done false
116752024-02-07T18:59:39.773Z 4 Acked 1003 Write 6 Done Done Done false
116762024-02-07T18:59:39.773Z 5 Acked 1004 Read 4 Done Done Done false
116772024-02-07T18:59:39.773Z 6 Acked 1005 Read 7 Done Done Done false
116782024-02-07T18:59:39.773Z 7 Acked 1006 Write 6 Done Done Done false
116792024-02-07T18:59:39.773Z 8 Acked 1007 Write 6 Done Done Done false
116802024-02-07T18:59:39.773Z 9 Acked 1008 Read 9 Done Done Done false
116812024-02-07T18:59:39.773Z 10 Acked 1009 Read 10 Done Done Done false
116822024-02-07T18:59:39.773Z 11 Acked 1010 Read 8 Done Done Done false
116832024-02-07T18:59:39.774Z 12 Acked 1011 Write 1 Done Done Done false
116842024-02-07T18:59:39.774Z 13 Acked 1012 Write 2 Done Sent Done false
116852024-02-07T18:59:39.774Z 14 Acked 1013 Read 6 Done Sent Done false
116862024-02-07T18:59:39.774Z 15 Acked 1014 Read 2 Done Sent Done false
116872024-02-07T18:59:39.774Z 16 Acked 1015 Write 2 Done Sent Done false
116882024-02-07T18:59:39.774Z 17 Acked 1016 Read 10 Done Sent Done false
116892024-02-07T18:59:39.774Z 18 Acked 1017 Read 6 Done Sent Done false
116902024-02-07T18:59:39.774Z 19 Acked 1018 Flush 0 Done Sent Done false
116912024-02-07T18:59:39.774Z 20 Acked 1019 Write 2 Done Sent Done false
116922024-02-07T18:59:39.775Z 21 Acked 1020 Read 1 Done Sent Done false
116932024-02-07T18:59:39.775Z 22 Acked 1021 Read 7 Done Sent Done false
116942024-02-07T18:59:39.775Z 23 Acked 1022 Write 5 Done Sent Done false
116952024-02-07T18:59:39.775Z 24 Acked 1023 Write 7 Done Sent Done false
116962024-02-07T18:59:39.775Z 25 Acked 1024 Write 4 Done Sent Done false
116972024-02-07T18:59:39.775Z 26 Acked 1025 Flush 0 Done Sent Done false
116982024-02-07T18:59:39.775Z 27 Acked 1026 Write 8 Done Sent Done false
116992024-02-07T18:59:39.775Z 28 Acked 1027 Write 6 Done Sent Done false
117002024-02-07T18:59:39.775Z 29 Acked 1028 Write 9 Sent Sent Sent false
117012024-02-07T18:59:39.775Z 30 Acked 1029 Write 4 Sent Sent Sent false
117022024-02-07T18:59:39.775Z STATES DS:0 DS:1 DS:2 TOTAL
117032024-02-07T18:59:39.775Z New 0 0 0 0
117042024-02-07T18:59:39.775Z Sent 2 18 2 22
117052024-02-07T18:59:39.776Z Done 26 10 26 62
117062024-02-07T18:59:39.776Z Skipped 0 0 0 0
117072024-02-07T18:59:39.776Z Error 0 0 0 0
117082024-02-07T18:59:39.776ZLast Flush: 1025 1001 1025
117092024-02-07T18:59:39.776ZDownstairs last five completed: 1001 1000
117102024-02-07T18:59:39.776ZUpstairs last five completed: 30 29 28 27 26
117112024-02-07T18:59:39.776Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:38.956188874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1559}
117122024-02-07T18:59:39.776ZWrote out file "/tmp/test_repair_verify.data"
117132024-02-07T18:59:39.776Z
117142024-02-07T18:59:39.776ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117152024-02-07T18:59:39.776ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117162024-02-07T18:59:39.776Z 0 000-019 104 104 104 183 183 183 F F F
117172024-02-07T18:59:39.777Z 1 020-039 102 102 102 175 175 175 F F F
117182024-02-07T18:59:39.777Z 2 040-059 104 104 104 178 178 178 F F F
117192024-02-07T18:59:39.777Z 3 060-079 106 91 106 187 150 187 F F F <---
117202024-02-07T18:59:39.777Z 4 080-099 106 86 106 187 141 187 F F F <---
117212024-02-07T18:59:39.777Z 5 100-119 101 101 101 173 173 173 F F F
117222024-02-07T18:59:39.777Z 6 120-139 103 103 103 177 177 177 F F F
117232024-02-07T18:59:39.777Z 7 140-159 100 100 100 172 172 172 F F F
117242024-02-07T18:59:39.777Z 8 160-179 99 99 99 165 165 165 F F F
117252024-02-07T18:59:39.777Z 9 180-199 105 105 105 184 184 184 F F F
117262024-02-07T18:59:39.777Z 10 200-219 106 100 106 187 167 187 F F F <---
117272024-02-07T18:59:39.778Z 11 220-239 106 106 106 185 185 185 F F F
117282024-02-07T18:59:39.778Z 12 240-259 104 104 104 180 180 180 F F F
117292024-02-07T18:59:39.778Z 13 260-279 102 102 102 175 175 175 F F F
117302024-02-07T18:59:39.778Z 14 280-299 104 104 104 183 183 183 T F T <---
117312024-02-07T18:59:39.778Z 15 300-319 96 96 96 161 161 161 T F T <---
117322024-02-07T18:59:39.778Z 16 320-339 106 104 106 186 178 186 F T F <---
117332024-02-07T18:59:39.778Z 17 340-359 106 104 106 186 178 186 F T F <---
117342024-02-07T18:59:39.778Z 18 360-379 99 99 99 165 165 165 F F F
117352024-02-07T18:59:39.778Z 19 380-399 105 105 105 184 184 184 F F F
117362024-02-07T18:59:39.778Z 20 400-419 105 105 105 184 184 184 F F F
117372024-02-07T18:59:39.779Z 21 420-439 102 102 102 176 176 176 F F F
117382024-02-07T18:59:39.779Z 22 440-459 102 102 102 175 175 175 F F F
117392024-02-07T18:59:39.779Z 23 460-479 106 102 106 186 175 186 F T F <---
117402024-02-07T18:59:39.779Z 24 480-499 104 104 104 180 180 180 F F F
117412024-02-07T18:59:39.779Z 25 500-519 106 100 106 186 169 186 F T F <---
117422024-02-07T18:59:39.779Z 26 520-539 94 94 94 157 157 157 F F F
117432024-02-07T18:59:39.779Z 27 540-559 106 103 106 187 177 187 F F F <---
117442024-02-07T18:59:39.779Z 28 560-579 106 104 106 186 182 186 F T F <---
117452024-02-07T18:59:39.779Z 29 580-599 104 104 104 179 179 179 T F T <---
117462024-02-07T18:59:39.779ZMax gen: 106, Max flush: 187
117472024-02-07T18:59:39.779ZError: Difference in extent metadata found!
117482024-02-07T18:59:39.780ZA Difference in extent metadata is expected here
117492024-02-07T18:59:39.780ZOn loop 53
117502024-02-07T18:59:39.780Z
117512024-02-07T18:59:39.780ZVerifying data now
117522024-02-07T18:59:46.593ZNo change after verify
117532024-02-07T18:59:46.593ZLoop: 53 Downstairs dump after verify (and repair):
117542024-02-07T18:59:46.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117552024-02-07T18:59:46.665Z 0 000-019 104 104 104 183 183 183 F F F
117562024-02-07T18:59:46.666Z 1 020-039 102 102 102 175 175 175 F F F
117572024-02-07T18:59:46.666Z 2 040-059 104 104 104 178 178 178 F F F
117582024-02-07T18:59:46.666Z 3 060-079 106 106 106 187 187 187 F F F
117592024-02-07T18:59:46.666Z 4 080-099 106 106 106 187 187 187 F F F
117602024-02-07T18:59:46.666Z 5 100-119 101 101 101 173 173 173 F F F
117612024-02-07T18:59:46.666Z 6 120-139 103 103 103 177 177 177 F F F
117622024-02-07T18:59:46.667Z 7 140-159 100 100 100 172 172 172 F F F
117632024-02-07T18:59:46.667Z 8 160-179 99 99 99 165 165 165 F F F
117642024-02-07T18:59:46.667Z 9 180-199 105 105 105 184 184 184 F F F
117652024-02-07T18:59:46.667Z 10 200-219 106 106 106 187 187 187 F F F
117662024-02-07T18:59:46.667Z 11 220-239 106 106 106 185 185 185 F F F
117672024-02-07T18:59:46.667Z 12 240-259 104 104 104 180 180 180 F F F
117682024-02-07T18:59:46.668Z 13 260-279 102 102 102 175 175 175 F F F
117692024-02-07T18:59:46.668Z 14 280-299 107 107 107 188 188 188 F F F
117702024-02-07T18:59:46.668Z 15 300-319 107 107 107 188 188 188 F F F
117712024-02-07T18:59:46.668Z 16 320-339 106 106 106 186 186 186 F F F
117722024-02-07T18:59:46.668Z 17 340-359 106 106 106 186 186 186 F F F
117732024-02-07T18:59:46.668Z 18 360-379 99 99 99 165 165 165 F F F
117742024-02-07T18:59:46.668Z 19 380-399 105 105 105 184 184 184 F F F
117752024-02-07T18:59:46.668Z 20 400-419 105 105 105 184 184 184 F F F
117762024-02-07T18:59:46.669Z 21 420-439 102 102 102 176 176 176 F F F
117772024-02-07T18:59:46.669Z 22 440-459 102 102 102 175 175 175 F F F
117782024-02-07T18:59:46.669Z 23 460-479 106 106 106 186 186 186 F F F
117792024-02-07T18:59:46.669Z 24 480-499 104 104 104 180 180 180 F F F
117802024-02-07T18:59:46.669Z 25 500-519 106 106 106 186 186 186 F F F
117812024-02-07T18:59:46.669Z 26 520-539 94 94 94 157 157 157 F F F
117822024-02-07T18:59:46.669Z 27 540-559 106 106 106 187 187 187 F F F
117832024-02-07T18:59:46.669Z 28 560-579 106 106 106 186 186 186 F F F
117842024-02-07T18:59:46.669Z 29 580-599 107 107 107 188 188 188 F F F
117852024-02-07T18:59:46.669ZMax gen: 107, Max flush: 188
117862024-02-07T18:59:46.669Z
117872024-02-07T18:59:46.670ZBegin loop 54
117882024-02-07T18:59:46.670ZDownstairs to restart: 1
117892024-02-07T18:59:46.670Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.61474462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567}
117902024-02-07T18:59:46.670Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615055568Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567}
117912024-02-07T18:59:46.670Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615082055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567}
117922024-02-07T18:59:46.670Z{"msg":"Crucible f12e85e7-de81-494e-9a54-3a88bf904479 has session id: fae9948a-c2a8-4797-a837-e0a00891338e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.61511903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
117932024-02-07T18:59:46.670Z{"msg":"Crucible runtime is spawned
117942024-02-07T18:59:46.670ZUpstairs opts: Upstairs UUID: f12e85e7-de81-494e-9a54-3a88bf904479, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615185601Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
117952024-02-07T18:59:46.670Z{"msg":"Crucible stats registered with UUID: f12e85e7-de81-494e-9a54-3a88bf904479","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615228755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
117962024-02-07T18:59:46.671Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615255131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567}
117972024-02-07T18:59:46.671Z{"msg":"f12e85e7-de81-494e-9a54-3a88bf904479 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615352708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
117982024-02-07T18:59:46.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615398172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
117992024-02-07T18:59:46.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615428498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118002024-02-07T18:59:46.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615455424Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118012024-02-07T18:59:46.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615497488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118022024-02-07T18:59:46.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615524115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118032024-02-07T18:59:46.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615550041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118042024-02-07T18:59:46.672Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615596485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118052024-02-07T18:59:46.672Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615638739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118062024-02-07T18:59:46.672Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615666296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118072024-02-07T18:59:46.672Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:44.615696352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118082024-02-07T18:59:46.672Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:44.615726888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118092024-02-07T18:59:46.672Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:44.615770851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118102024-02-07T18:59:46.672Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615796328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118112024-02-07T18:59:46.672Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615842472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118122024-02-07T18:59:46.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615868418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118132024-02-07T18:59:46.673Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615907293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118142024-02-07T18:59:46.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.615935569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118152024-02-07T18:59:46.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.616458189Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118162024-02-07T18:59:46.673Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.616489324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118172024-02-07T18:59:46.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.616504442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118182024-02-07T18:59:46.673Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.616532468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118192024-02-07T18:59:46.674Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.617055848Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118202024-02-07T18:59:46.674Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.61718731Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118212024-02-07T18:59:46.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.617573668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118222024-02-07T18:59:46.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.617621731Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118232024-02-07T18:59:46.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.617649168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118242024-02-07T18:59:46.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.617664126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118252024-02-07T18:59:46.674Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.617678813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118262024-02-07T18:59:46.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:44.61770192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118272024-02-07T18:59:46.675Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.617426807Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118282024-02-07T18:59:46.675Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.617560659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118292024-02-07T18:59:46.675Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.618349362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118302024-02-07T18:59:46.675Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.618394706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118312024-02-07T18:59:46.675Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.618934963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118322024-02-07T18:59:46.675Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619531342Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118332024-02-07T18:59:46.676Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619584875Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118342024-02-07T18:59:46.676Z{"msg":"[0]R flush_numbers[0..12]: [183, 175, 178, 187, 187, 173, 177, 172, 165, 184, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619608542Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118352024-02-07T18:59:46.676Z{"msg":"[0]R generation[0..12]: [104, 102, 104, 106, 106, 101, 103, 100, 99, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619637908Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118362024-02-07T18:59:46.676Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619657395Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118372024-02-07T18:59:46.676Z{"msg":"[1]R flush_numbers[0..12]: [183, 175, 178, 187, 187, 173, 177, 172, 165, 184, 187, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619683862Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118382024-02-07T18:59:46.676Z{"msg":"[1]R generation[0..12]: [104, 102, 104, 106, 106, 101, 103, 100, 99, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619702629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118392024-02-07T18:59:46.676Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619720577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118402024-02-07T18:59:46.676Z{"msg":"[2]R flush_numbers[0..12]: [183, 175, 178, 187, 187, 173, 177, 172, 165, 184, 187, 185]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
118412024-02-07T18:59:46.677Z,"time":"2024-02-07T18:59:45.619740134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118422024-02-07T18:59:46.677Z{"msg":"[2]R generation[0..12]: [104, 102, 104, 106, 106, 101, 103, 100, 99, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619761061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118432024-02-07T18:59:46.677Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619779449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118442024-02-07T18:59:46.677Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619805205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118452024-02-07T18:59:46.677Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619823303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118462024-02-07T18:59:46.677Z{"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619854648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118472024-02-07T18:59:46.677Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619872046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118482024-02-07T18:59:46.677Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619889044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118492024-02-07T18:59:46.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619912651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118502024-02-07T18:59:46.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619929648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118512024-02-07T18:59:46.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619953825Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118522024-02-07T18:59:46.678Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.619970723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118532024-02-07T18:59:46.678Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.620011837Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118542024-02-07T18:59:46.772Z{"msg":"f12e85e7-de81-494e-9a54-3a88bf904479 is now active with session: fae9948a-c2a8-4797-a837-e0a00891338e","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.620039973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118552024-02-07T18:59:46.773Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.620055341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567,"session_id":"fae9948a-c2a8-4797-a837-e0a00891338e"}
118562024-02-07T18:59:46.773Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:45.620079278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567}
118572024-02-07T18:59:46.773ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118582024-02-07T18:59:46.773ZLoading write count information from file "/tmp/test_repair_verify.data"
118592024-02-07T18:59:46.773ZRun Repair workload
118602024-02-07T18:59:46.773Z01/30 Write block 25 len 12288 data: 8 6 6
118612024-02-07T18:59:46.773Z02/30 Read block 40 len 40960
118622024-02-07T18:59:46.773Z03/30 Read block 324 len 24576
118632024-02-07T18:59:46.773Z04/30 Flush
118642024-02-07T18:59:46.774Z05/30 Write block 580 len 4096 data: 7
118652024-02-07T18:59:46.774Z06/30 Write block 220 len 8192 data: 6 6
118662024-02-07T18:59:46.774Z07/30 Write block 225 len 16384 data: 6 4 4 5
118672024-02-07T18:59:46.774Z08/30 Read block 374 len 40960
118682024-02-07T18:59:46.774Z09/30 Write block 373 len 32768 data: 8 7 8 8 7 8 6 7
118692024-02-07T18:59:46.774Z10/30 Write block 122 len 36864 data: 12 10 8 8 8 8 10 10 8
118702024-02-07T18:59:46.774Z11/30 Flush
118712024-02-07T18:59:46.774Z12/30 Write block 287 len 28672 data: 16 16 15 16 14 11 9
118722024-02-07T18:59:46.775Z13/30 Write block 486 len 36864 data: 6 7 7 5 6 6 6 7 9
118732024-02-07T18:59:46.775Z14/30 Read block 121 len 24576
118742024-02-07T18:59:46.775Z15/30 Read block 129 len 16384
118752024-02-07T18:59:46.775Z16/30 Read block 299 len 36864
118762024-02-07T18:59:46.775Z17/30 Read block 541 len 4096
118772024-02-07T18:59:46.775Z18/30 Read block 486 len 16384
118782024-02-07T18:59:46.775Z19/30 Write block 79 len 4096 data: 8
118792024-02-07T18:59:46.775Z20/30 Write block 345 len 16384 data: 11 13 13 12
118802024-02-07T18:59:46.775Z21/30 Write block 496 len 20480 data: 8 6 7 6 7
118812024-02-07T18:59:46.775Z22/30 Read block 581 len 20480
118822024-02-07T18:59:46.775Z23/30 Flush
118832024-02-07T18:59:46.775Z24/30 Write block 396 len 16384 data: 7 8 8 10
118842024-02-07T18:59:46.775Z25/30 Read block 394 len 20480
118852024-02-07T18:59:46.775Z26/30 Read block 134 len 12288
118862024-02-07T18:59:46.775Z27/30 Read block 295 len 12288
118872024-02-07T18:59:46.776Z28/30 Read block 228 len 32768
118882024-02-07T18:59:46.776Z29/30 Read block 11 len 40960
118892024-02-07T18:59:46.776Z30/30 Write block 100 len 32768 data: 6 6 6 7 10 12 11 12
118902024-02-07T18:59:46.776Z----------------------------------------------------------------
118912024-02-07T18:59:46.776Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:26
118922024-02-07T18:59:46.776ZStates: Active Active Active
118932024-02-07T18:59:46.776ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118942024-02-07T18:59:46.776Z 5 Acked 1004 Write 1 Done Sent Done false
118952024-02-07T18:59:46.777Z 6 Acked 1005 Write 2 Done Sent Done false
118962024-02-07T18:59:46.777Z 7 Acked 1006 Write 4 Done Sent Done false
118972024-02-07T18:59:46.777Z 8 Acked 1007 Read 10 Done Sent Done false
118982024-02-07T18:59:46.777Z 9 Acked 1008 Write 8 Done Sent Done false
118992024-02-07T18:59:46.777Z 10 Acked 1009 Write 9 Done Sent Done false
119002024-02-07T18:59:46.777Z 11 Acked 1010 Flush 0 Done Sent Done false
119012024-02-07T18:59:46.777Z 12 Acked 1011 Write 7 Done Sent Done false
119022024-02-07T18:59:46.777Z 13 Acked 1012 Write 9 Done Sent Done false
119032024-02-07T18:59:46.777Z 14 Acked 1013 Read 6 Done Sent Done false
119042024-02-07T18:59:46.778Z 15 Acked 1014 Read 4 Done Sent Done false
119052024-02-07T18:59:46.778Z 16 Acked 1015 Read 9 Done Sent Done false
119062024-02-07T18:59:46.778Z 17 Acked 1016 Read 1 Done Sent Done false
119072024-02-07T18:59:46.778Z 18 Acked 1017 Read 4 Done Sent Done false
119082024-02-07T18:59:46.778Z 19 Acked 1018 Write 1 Done Sent Done false
119092024-02-07T18:59:46.778Z 20 Acked 1019 Write 4 Done Sent Done false
119102024-02-07T18:59:46.778Z 21 Acked 1020 Write 5 Done Sent Done false
119112024-02-07T18:59:46.778Z 22 Acked 1021 Read 5 Done Sent Done false
119122024-02-07T18:59:46.779Z 23 Acked 1022 Flush 0 Done Sent Done false
119132024-02-07T18:59:46.779Z 24 Acked 1023 Write 4 Done Sent Done false
119142024-02-07T18:59:46.779Z 25 Acked 1024 Read 5 Done Sent Done false
119152024-02-07T18:59:46.779Z 26 Acked 1025 Read 3 Done Sent Done false
119162024-02-07T18:59:46.779Z 27 Acked 1026 Read 3 Done Sent Done false
119172024-02-07T18:59:46.779Z 28 Acked 1027 Read 8 Done Sent Done false
119182024-02-07T18:59:46.779Z 29 Acked 1028 Read 10 Done Sent Done false
119192024-02-07T18:59:46.779Z 30 Acked 1029 Write 8 Sent Sent Sent false
119202024-02-07T18:59:46.780Z STATES DS:0 DS:1 DS:2 TOTAL
119212024-02-07T18:59:46.780Z New 0 0 0 0
119222024-02-07T18:59:46.780Z Sent 1 26 1 28
119232024-02-07T18:59:46.780Z Done 25 0 25 50
119242024-02-07T18:59:46.780Z Skipped 0 0 0 0
119252024-02-07T18:59:46.780Z Error 0 0 0 0
119262024-02-07T18:59:46.780ZLast Flush: 1022 1003 1022
119272024-02-07T18:59:46.781ZDownstairs last five completed: 1003 1002 1001 1000
119282024-02-07T18:59:46.781ZUpstairs last five completed: 30 29 28 27 26
119292024-02-07T18:59:46.781Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:45.651542403Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1567}
119302024-02-07T18:59:46.781ZWrote out file "/tmp/test_repair_verify.data"
119312024-02-07T18:59:46.781Z
119322024-02-07T18:59:46.781ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119332024-02-07T18:59:46.781ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119342024-02-07T18:59:46.782Z 0 000-019 104 104 104 183 183 183 F F F
119352024-02-07T18:59:46.782Z 1 020-039 108 108 108 189 189 189 F F F
119362024-02-07T18:59:46.782Z 2 040-059 104 104 104 178 178 178 F F F
119372024-02-07T18:59:46.782Z 3 060-079 108 106 108 191 187 191 F F F <---
119382024-02-07T18:59:46.782Z 4 080-099 106 106 106 187 187 187 F F F
119392024-02-07T18:59:46.782Z 5 100-119 101 101 101 173 173 173 T F T <---
119402024-02-07T18:59:46.782Z 6 120-139 108 103 108 190 177 190 F F F <---
119412024-02-07T18:59:46.782Z 7 140-159 100 100 100 172 172 172 F F F
119422024-02-07T18:59:46.782Z 8 160-179 99 99 99 165 165 165 F F F
119432024-02-07T18:59:46.783Z 9 180-199 105 105 105 184 184 184 F F F
119442024-02-07T18:59:46.783Z 10 200-219 106 106 106 187 187 187 F F F
119452024-02-07T18:59:46.783Z 11 220-239 108 106 108 190 185 190 F F F <---
119462024-02-07T18:59:46.783Z 12 240-259 104 104 104 180 180 180 F F F
119472024-02-07T18:59:46.783Z 13 260-279 102 102 102 175 175 175 F F F
119482024-02-07T18:59:46.783Z 14 280-299 108 107 108 191 188 191 F F F <---
119492024-02-07T18:59:46.783Z 15 300-319 107 107 107 188 188 188 F F F
119502024-02-07T18:59:46.783Z 16 320-339 106 106 106 186 186 186 F F F
119512024-02-07T18:59:46.783Z 17 340-359 108 106 108 191 186 191 F F F <---
119522024-02-07T18:59:46.783Z 18 360-379 108 99 108 190 165 190 F F F <---
119532024-02-07T18:59:46.784Z 19 380-399 108 105 108 190 184 190 T F T <---
119542024-02-07T18:59:46.801Z 20 400-419 105 105 105 184 184 184 F F F
119552024-02-07T18:59:46.801Z 21 420-439 102 102 102 176 176 176 F F F
119562024-02-07T18:59:46.801Z 22 440-459 102 102 102 175 175 175 F F F
119572024-02-07T18:59:46.801Z 23 460-479 106 106 106 186 186 186 F F F
119582024-02-07T18:59:46.801Z 24 480-499 108 104 108 191 180 191 F F F <---
119592024-02-07T18:59:46.801Z 25 500-519 108 106 108 191 186 191 F F F <---
119602024-02-07T18:59:46.801Z 26 520-539 94 94 94 157 157 157 F F F
119612024-02-07T18:59:46.802Z 27 540-559 106 106 106 187 187 187 F F F
119622024-02-07T18:59:46.802Z 28 560-579 106 106 106 186 186 186 F F F
119632024-02-07T18:59:46.802Z 29 580-599 108 107 108 190 188 190 F F F <---
119642024-02-07T18:59:46.802ZMax gen: 108, Max flush: 191
119652024-02-07T18:59:46.802ZError: Difference in extent metadata found!
119662024-02-07T18:59:46.802ZA Difference in extent metadata is expected here
119672024-02-07T18:59:46.802ZOn loop 54
119682024-02-07T18:59:46.803Z
119692024-02-07T18:59:46.803ZVerifying data now
119702024-02-07T18:59:51.010ZNo change after verify
119712024-02-07T18:59:51.011ZLoop: 54 Downstairs dump after verify (and repair):
119722024-02-07T18:59:51.030ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119732024-02-07T18:59:51.030Z 0 000-019 104 104 104 183 183 183 F F F
119742024-02-07T18:59:51.030Z 1 020-039 108 108 108 189 189 189 F F F
119752024-02-07T18:59:51.031Z 2 040-059 104 104 104 178 178 178 F F F
119762024-02-07T18:59:51.031Z 3 060-079 108 108 108 191 191 191 F F F
119772024-02-07T18:59:51.031Z 4 080-099 106 106 106 187 187 187 F F F
119782024-02-07T18:59:51.031Z 5 100-119 109 109 109 192 192 192 F F F
119792024-02-07T18:59:51.031Z 6 120-139 108 108 108 190 190 190 F F F
119802024-02-07T18:59:51.031Z 7 140-159 100 100 100 172 172 172 F F F
119812024-02-07T18:59:51.032Z 8 160-179 99 99 99 165 165 165 F F F
119822024-02-07T18:59:51.042Z 9 180-199 105 105 105 184 184 184 F F F
119832024-02-07T18:59:51.042Z 10 200-219 106 106 106 187 187 187 F F F
119842024-02-07T18:59:51.042Z 11 220-239 108 108 108 190 190 190 F F F
119852024-02-07T18:59:51.042Z 12 240-259 104 104 104 180 180 180 F F F
119862024-02-07T18:59:51.042Z 13 260-279 102 102 102 175 175 175 F F F
119872024-02-07T18:59:51.043Z 14 280-299 108 108 108 191 191 191 F F F
119882024-02-07T18:59:51.043Z 15 300-319 107 107 107 188 188 188 F F F
119892024-02-07T18:59:51.043Z 16 320-339 106 106 106 186 186 186 F F F
119902024-02-07T18:59:51.043Z 17 340-359 108 108 108 191 191 191 F F F
119912024-02-07T18:59:51.044Z 18 360-379 108 108 108 190 190 190 F F F
119922024-02-07T18:59:51.044Z 19 380-399 109 109 109 192 192 192 F F F
119932024-02-07T18:59:51.044Z 20 400-419 105 105 105 184 184 184 F F F
119942024-02-07T18:59:51.044Z 21 420-439 102 102 102 176 176 176 F F F
119952024-02-07T18:59:51.045Z 22 440-459 102 102 102 175 175 175 F F F
119962024-02-07T18:59:51.045Z 23 460-479 106 106 106 186 186 186 F F F
119972024-02-07T18:59:51.045Z 24 480-499 108 108 108 191 191 191 F F F
119982024-02-07T18:59:51.045Z 25 500-519 108 108 108 191 191 191 F F F
119992024-02-07T18:59:51.045Z 26 520-539 94 94 94 157 157 157 F F F
120002024-02-07T18:59:51.046Z 27 540-559 106 106 106 187 187 187 F F F
120012024-02-07T18:59:51.046Z 28 560-579 106 106 106 186 186 186 F F F
120022024-02-07T18:59:51.046Z 29 580-599 108 108 108 190 190 190 F F F
120032024-02-07T18:59:51.046ZMax gen: 109, Max flush: 192
120042024-02-07T18:59:51.046Z
120052024-02-07T18:59:51.047ZBegin loop 55
120062024-02-07T18:59:51.047ZDownstairs to restart: 0
120072024-02-07T18:59:51.074Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.26706981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575}
120082024-02-07T18:59:51.074Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267320926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575}
120092024-02-07T18:59:51.074Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267352812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575}
120102024-02-07T18:59:51.074Z{"msg":"Crucible 370b7caa-e591-413f-ad53-a400fd4d476f has session id: bae235b8-8685-4ace-b79e-ee8488f90ad4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267387147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120112024-02-07T18:59:51.074Z{"msg":"Crucible runtime is spawned
120122024-02-07T18:59:51.074ZUpstairs opts: Upstairs UUID: 370b7caa-e591-413f-ad53-a400fd4d476f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267443439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120132024-02-07T18:59:51.075Z{"msg":"Crucible stats registered with UUID: 370b7caa-e591-413f-ad53-a400fd4d476f","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267488173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120142024-02-07T18:59:51.075Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267523149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575}
120152024-02-07T18:59:51.075Z{"msg":"370b7caa-e591-413f-ad53-a400fd4d476f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267683707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120162024-02-07T18:59:51.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267718132Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120172024-02-07T18:59:51.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.2677352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120182024-02-07T18:59:51.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267749738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120192024-02-07T18:59:51.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267764046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120202024-02-07T18:59:51.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267778474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120212024-02-07T18:59:51.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.26780637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120222024-02-07T18:59:51.075Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267821518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120232024-02-07T18:59:51.075Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267840406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120242024-02-07T18:59:51.075Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.267873281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120252024-02-07T18:59:51.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268002834Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120262024-02-07T18:59:51.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268054187Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120272024-02-07T18:59:51.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268265258Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120282024-02-07T18:59:51.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268785528Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120292024-02-07T18:59:51.076Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268832162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120302024-02-07T18:59:51.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268863797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120312024-02-07T18:59:51.076Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.268926879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120322024-02-07T18:59:51.076Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.269601687Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120332024-02-07T18:59:51.076Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.269638582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120342024-02-07T18:59:51.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.269664769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120352024-02-07T18:59:51.076Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.269690015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120362024-02-07T18:59:51.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270291664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120372024-02-07T18:59:51.076Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270328579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120382024-02-07T18:59:51.076Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270344477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120392024-02-07T18:59:51.077Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270358805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120402024-02-07T18:59:51.077Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270380192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120412024-02-07T18:59:51.077Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.27039427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120422024-02-07T18:59:51.077Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270408858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120432024-02-07T18:59:51.077Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270874975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120442024-02-07T18:59:51.077Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.27091339Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120452024-02-07T18:59:51.078Z{"msg":"[0]R flush_numbers[0..12]: [183, 189, 178, 191, 187, 192, 190, 172, 165, 184, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270929948Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120462024-02-07T18:59:51.078Z{"msg":"[0]R generation[0..12]: [104, 108, 104, 108, 106, 109, 108, 100, 99, 105, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270946106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120472024-02-07T18:59:51.078Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270968683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120482024-02-07T18:59:51.078Z{"msg":"[1]R flush_numbers[0..12]: [183, 189, 178, 191, 187, 192, 190, 172, 165, 184, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.270983651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120492024-02-07T18:59:51.079Z{"msg":"[1]R generation[0..12]: [104, 108, 104, 108, 106, 109, 108, 100, 99, 105, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271005298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120502024-02-07T18:59:51.079Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271020795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120512024-02-07T18:59:51.079Z{"msg":"[2]R flush_numbers[0..12]: [183, 189, 178, 191, 187, 192, 190, 172, 165, 184, 187, 190]",Wait for a query_work_queue command to finish before sending IO
120522024-02-07T18:59:51.079Z"v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271046662Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120532024-02-07T18:59:51.080Z{"msg":"[2]R generation[0..12]: [104, 108, 104, 108, 106, 109, 108, 100, 99, 105, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.27106139Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120542024-02-07T18:59:51.080Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271075718Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120552024-02-07T18:59:51.080Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271089766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120562024-02-07T18:59:51.080Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271110823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120572024-02-07T18:59:51.080Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271125241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120582024-02-07T18:59:51.081Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271145609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120592024-02-07T18:59:51.081Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271159657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120602024-02-07T18:59:51.081Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271173715Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120612024-02-07T18:59:51.081Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271195412Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120622024-02-07T18:59:51.081Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.27120934Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120632024-02-07T18:59:51.082Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271230047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120642024-02-07T18:59:51.082Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271243605Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120652024-02-07T18:59:51.082Z{"msg":"370b7caa-e591-413f-ad53-a400fd4d476f is now active with session: bae235b8-8685-4ace-b79e-ee8488f90ad4","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271256624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120662024-02-07T18:59:51.082Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271277121Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575,"session_id":"bae235b8-8685-4ace-b79e-ee8488f90ad4"}
120672024-02-07T18:59:51.083Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:50.271290009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575}
120682024-02-07T18:59:51.083ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120692024-02-07T18:59:51.083ZLoading write count information from file "/tmp/test_repair_verify.data"
120702024-02-07T18:59:51.083ZRun Repair workload
120712024-02-07T18:59:51.083Z01/30 Write block 449 len 32768 data: 8 8 9 8 10 9 8 7
120722024-02-07T18:59:51.084Z02/30 Read block 175 len 32768
120732024-02-07T18:59:51.084Z03/30 Write block 566 len 12288 data: 10 11 11
120742024-02-07T18:59:51.084Z04/30 Write block 529 len 20480 data: 9 8 5 7 8
120752024-02-07T18:59:51.084Z05/30 Flush
120762024-02-07T18:59:51.085Z06/30 Write block 5 len 28672 data: 9 8 11 12 13 9 9
120772024-02-07T18:59:51.085Z07/30 Read block 450 len 16384
120782024-02-07T18:59:51.085Z08/30 Write block 197 len 8192 data: 9 11
120792024-02-07T18:59:51.085Z09/30 Write block 421 len 4096 data: 7
120802024-02-07T18:59:51.523Z10/30 Read block 278 len 40960
120812024-02-07T18:59:51.523Z11/30 Read block 207 len 36864
120822024-02-07T18:59:51.523Z12/30 Read block 540 len 24576
120832024-02-07T18:59:51.523Z13/30 Read block 153 len 12288
120842024-02-07T18:59:51.523Z14/30 Read block 249 len 24576
120852024-02-07T18:59:51.523Z15/30 Write block 123 len 20480 data: 11 9 9 9 9
120862024-02-07T18:59:51.523Z16/30 Read block 113 len 36864
120872024-02-07T18:59:51.524Z17/30 Read block 508 len 8192
120882024-02-07T18:59:51.524Z18/30 Write block 191 len 20480 data: 9 10 9 10 8
120892024-02-07T18:59:51.524Z19/30 Read block 521 len 24576
120902024-02-07T18:59:51.524Z20/30 Read block 294 len 24576
120912024-02-07T18:59:51.524Z21/30 Read block 229 len 40960
120922024-02-07T18:59:51.524Z22/30 Read block 341 len 16384
120932024-02-07T18:59:51.524Z23/30 Read block 169 len 32768
120942024-02-07T18:59:51.524Z24/30 Read block 226 len 32768
120952024-02-07T18:59:51.524Z25/30 Read block 344 len 12288
120962024-02-07T18:59:51.524Z26/30 Read block 486 len 40960
120972024-02-07T18:59:51.524Z27/30 Read block 384 len 12288
120982024-02-07T18:59:51.524Z28/30 Write block 179 len 8192 data: 6 8
120992024-02-07T18:59:51.524Z29/30 Write block 21 len 36864 data: 12 12 12 10 9 7 7 9 11
121002024-02-07T18:59:51.525Z30/30 Write block 267 len 4096 data: 11
121012024-02-07T18:59:51.525Z----------------------------------------------------------------
121022024-02-07T18:59:51.525Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
121032024-02-07T18:59:51.525ZStates: Active Active Active
121042024-02-07T18:59:51.525ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121052024-02-07T18:59:51.525Z 1 Acked 1000 Write 8 Sent Done Done false
121062024-02-07T18:59:51.525Z 2 Acked 1001 Read 8 Sent Done Done false
121072024-02-07T18:59:51.525Z 3 Acked 1002 Write 3 Sent Done Done false
121082024-02-07T18:59:51.525Z 4 Acked 1003 Write 5 Sent Done Done false
121092024-02-07T18:59:51.525Z 5 Acked 1004 Flush 0 Sent Done Done false
121102024-02-07T18:59:51.525Z 6 Acked 1005 Write 7 Sent Done Done false
121112024-02-07T18:59:51.525Z 7 Acked 1006 Read 4 Sent Done Done false
121122024-02-07T18:59:51.526Z 8 Acked 1007 Write 2 Sent Done Done false
121132024-02-07T18:59:51.526Z 9 Acked 1008 Write 1 Sent Done Done false
121142024-02-07T18:59:51.526Z 10 Acked 1009 Read 10 Sent Done Done false
121152024-02-07T18:59:51.526Z 11 Acked 1010 Read 9 Sent Done Done false
121162024-02-07T18:59:51.526Z 12 Acked 1011 Read 6 Sent Done Done false
121172024-02-07T18:59:51.526Z 13 Acked 1012 Read 3 Sent Done Done false
121182024-02-07T18:59:51.527Z 14 Acked 1013 Read 6 Sent Done Done false
121192024-02-07T18:59:51.527Z 15 Acked 1014 Write 5 Sent Done Done false
121202024-02-07T18:59:51.527Z 16 Acked 1015 Read 9 Sent Done Done false
121212024-02-07T18:59:51.527Z 17 Acked 1016 Read 2 Sent Done Done false
121222024-02-07T18:59:51.527Z 18 Acked 1017 Write 5 Sent Done Done false
121232024-02-07T18:59:51.527Z 19 Acked 1018 Read 6 Sent Done Done false
121242024-02-07T18:59:51.527Z 20 Acked 1019 Read 6 Sent Done Done false
121252024-02-07T18:59:51.527Z 21 Acked 1020 Read 10 Sent Done Done false
121262024-02-07T18:59:51.527Z 22 Acked 1021 Read 4 Sent Done Done false
121272024-02-07T18:59:51.527Z 23 Acked 1022 Read 8 Sent Done Done false
121282024-02-07T18:59:51.528Z 24 Acked 1023 Read 8 Sent Done Done false
121292024-02-07T18:59:51.528Z 25 Acked 1024 Read 3 Sent Done Done false
121302024-02-07T18:59:51.528Z 26 Acked 1025 Read 10 Sent Done Done false
121312024-02-07T18:59:51.528Z 27 Acked 1026 Read 3 Sent Done Done false
121322024-02-07T18:59:51.528Z 28 Acked 1027 Write 2 Sent Done Done false
121332024-02-07T18:59:51.528Z 29 Acked 1028 Write 9 Sent Sent Sent false
121342024-02-07T18:59:51.528Z 30 Acked 1029 Write 1 Sent Sent Sent false
121352024-02-07T18:59:51.528Z STATES DS:0 DS:1 DS:2 TOTAL
121362024-02-07T18:59:51.528Z New 0 0 0 0
121372024-02-07T18:59:51.528Z Sent 30 2 2 34
121382024-02-07T18:59:51.528Z Done 0 28 28 56
121392024-02-07T18:59:51.528Z Skipped 0 0 0 0
121402024-02-07T18:59:51.528Z Error 0 0 0 0
121412024-02-07T18:59:51.529ZLast Flush: 0 1004 1004
121422024-02-07T18:59:51.529ZDownstairs last five completed:
121432024-02-07T18:59:51.529ZUpstairs last five completed: 30 29 28 27 26
121442024-02-07T18:59:51.529Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:50.299787095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1575}
121452024-02-07T18:59:51.529ZWrote out file "/tmp/test_repair_verify.data"
121462024-02-07T18:59:51.529Z
121472024-02-07T18:59:51.529ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121482024-02-07T18:59:51.536ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121492024-02-07T18:59:51.536Z 0 000-019 104 104 104 183 183 183 F T T <---
121502024-02-07T18:59:51.536Z 1 020-039 108 108 108 189 189 189 F T T <---
121512024-02-07T18:59:51.536Z 2 040-059 104 104 104 178 178 178 F F F
121522024-02-07T18:59:51.537Z 3 060-079 108 108 108 191 191 191 F F F
121532024-02-07T18:59:51.537Z 4 080-099 106 106 106 187 187 187 F F F
121542024-02-07T18:59:51.537Z 5 100-119 109 109 109 192 192 192 F F F
121552024-02-07T18:59:51.539Z 6 120-139 108 108 108 190 190 190 F T T <---
121562024-02-07T18:59:51.539Z 7 140-159 100 100 100 172 172 172 F F F
121572024-02-07T18:59:51.539Z 8 160-179 99 99 99 165 165 165 F T T <---
121582024-02-07T18:59:51.540Z 9 180-199 105 105 105 184 184 184 F T T <---
121592024-02-07T18:59:51.540Z 10 200-219 106 106 106 187 187 187 F F F
121602024-02-07T18:59:51.540Z 11 220-239 108 108 108 190 190 190 F F F
121612024-02-07T18:59:51.540Z 12 240-259 104 104 104 180 180 180 F F F
121622024-02-07T18:59:51.540Z 13 260-279 102 102 102 175 175 175 F T T <---
121632024-02-07T18:59:51.540Z 14 280-299 108 108 108 191 191 191 F F F
121642024-02-07T18:59:51.540Z 15 300-319 107 107 107 188 188 188 F F F
121652024-02-07T18:59:51.540Z 16 320-339 106 106 106 186 186 186 F F F
121662024-02-07T18:59:51.540Z 17 340-359 108 108 108 191 191 191 F F F
121672024-02-07T18:59:51.540Z 18 360-379 108 108 108 190 190 190 F F F
121682024-02-07T18:59:51.540Z 19 380-399 109 109 109 192 192 192 F F F
121692024-02-07T18:59:51.540Z 20 400-419 105 105 105 184 184 184 F F F
121702024-02-07T18:59:51.540Z 21 420-439 102 102 102 176 176 176 F T T <---
121712024-02-07T18:59:51.540Z 22 440-459 102 110 110 175 193 193 F F F <---
121722024-02-07T18:59:51.541Z 23 460-479 106 106 106 186 186 186 F F F
121732024-02-07T18:59:51.541Z 24 480-499 108 108 108 191 191 191 F F F
121742024-02-07T18:59:51.541Z 25 500-519 108 108 108 191 191 191 F F F
121752024-02-07T18:59:51.541Z 26 520-539 94 110 110 157 193 193 F F F <---
121762024-02-07T18:59:51.541Z 27 540-559 106 106 106 187 187 187 F F F
121772024-02-07T18:59:51.541Z 28 560-579 106 110 110 186 193 193 F F F <---
121782024-02-07T18:59:51.541Z 29 580-599 108 108 108 190 190 190 F F F
121792024-02-07T18:59:51.541ZMax gen: 110, Max flush: 193
121802024-02-07T18:59:51.541ZError: Difference in extent metadata found!
121812024-02-07T18:59:51.541ZA Difference in extent metadata is expected here
121822024-02-07T18:59:51.541ZOn loop 55
121832024-02-07T18:59:51.541Z
121842024-02-07T18:59:51.541ZVerifying data now
121852024-02-07T18:59:55.628ZNo change after verify
121862024-02-07T18:59:55.628ZLoop: 55 Downstairs dump after verify (and repair):
121872024-02-07T18:59:55.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121882024-02-07T18:59:55.644Z 0 000-019 111 111 111 194 194 194 F F F
121892024-02-07T18:59:55.645Z 1 020-039 111 111 111 194 194 194 F F F
121902024-02-07T18:59:55.645Z 2 040-059 104 104 104 178 178 178 F F F
121912024-02-07T18:59:55.645Z 3 060-079 108 108 108 191 191 191 F F F
121922024-02-07T18:59:55.645Z 4 080-099 106 106 106 187 187 187 F F F
121932024-02-07T18:59:55.645Z 5 100-119 109 109 109 192 192 192 F F F
121942024-02-07T18:59:55.645Z 6 120-139 111 111 111 194 194 194 F F F
121952024-02-07T18:59:55.646Z 7 140-159 100 100 100 172 172 172 F F F
121962024-02-07T18:59:55.646Z 8 160-179 111 111 111 194 194 194 F F F
121972024-02-07T18:59:55.646Z 9 180-199 111 111 111 194 194 194 F F F
121982024-02-07T18:59:55.650Z 10 200-219 106 106 106 187 187 187 F F F
121992024-02-07T18:59:55.650Z 11 220-239 108 108 108 190 190 190 F F F
122002024-02-07T18:59:55.650Z 12 240-259 104 104 104 180 180 180 F F F
122012024-02-07T18:59:55.650Z 13 260-279 111 111 111 194 194 194 F F F
122022024-02-07T18:59:55.651Z 14 280-299 108 108 108 191 191 191 F F F
122032024-02-07T18:59:55.651Z 15 300-319 107 107 107 188 188 188 F F F
122042024-02-07T18:59:55.651Z 16 320-339 106 106 106 186 186 186 F F F
122052024-02-07T18:59:55.651Z 17 340-359 108 108 108 191 191 191 F F F
122062024-02-07T18:59:55.651Z 18 360-379 108 108 108 190 190 190 F F F
122072024-02-07T18:59:55.651Z 19 380-399 109 109 109 192 192 192 F F F
122082024-02-07T18:59:55.651Z 20 400-419 105 105 105 184 184 184 F F F
122092024-02-07T18:59:55.652Z 21 420-439 111 111 111 194 194 194 F F F
122102024-02-07T18:59:55.652Z 22 440-459 110 110 110 193 193 193 F F F
122112024-02-07T18:59:55.652Z 23 460-479 106 106 106 186 186 186 F F F
122122024-02-07T18:59:55.652Z 24 480-499 108 108 108 191 191 191 F F F
122132024-02-07T18:59:55.652Z 25 500-519 108 108 108 191 191 191 F F F
122142024-02-07T18:59:55.652Z 26 520-539 110 110 110 193 193 193 F F F
122152024-02-07T18:59:55.652Z 27 540-559 106 106 106 187 187 187 F F F
122162024-02-07T18:59:55.652Z 28 560-579 110 110 110 193 193 193 F F F
122172024-02-07T18:59:55.653Z 29 580-599 108 108 108 190 190 190 F F F
122182024-02-07T18:59:55.653ZMax gen: 111, Max flush: 194
122192024-02-07T18:59:55.653Z
122202024-02-07T18:59:55.653ZBegin loop 56
122212024-02-07T18:59:55.653ZDownstairs to restart: 1
122222024-02-07T18:59:55.665Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.891758997Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583}
122232024-02-07T18:59:55.666Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892008804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583}
122242024-02-07T18:59:55.666Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892031431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583}
122252024-02-07T18:59:55.672Z{"msg":"Crucible 2f4feab6-b542-436a-95c6-cbfe079d7974 has session id: 8861fa5f-081e-43fb-acb5-0116d184467c","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892055157Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122262024-02-07T18:59:55.672Z{"msg":"Upstairs opts: Upstairs UUID: 2f4feab6-b542-436a-95c6-cbfe079d7974, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.89211216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122272024-02-07T18:59:55.672Z{"msg":"Crucible runtime is spawned
122282024-02-07T18:59:55.672ZCrucible stats registered with UUID: 2f4feab6-b542-436a-95c6-cbfe079d7974","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892146315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122292024-02-07T18:59:55.673Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892248441Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583}
122302024-02-07T18:59:55.673Z{"msg":"2f4feab6-b542-436a-95c6-cbfe079d7974 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892357287Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122312024-02-07T18:59:55.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892389322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122322024-02-07T18:59:55.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.89240488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122332024-02-07T18:59:55.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892419538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122342024-02-07T18:59:55.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892452554Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122352024-02-07T18:59:55.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.89247774Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122362024-02-07T18:59:55.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892515935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122372024-02-07T18:59:55.674Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.8925518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122382024-02-07T18:59:55.674Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892578357Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122392024-02-07T18:59:55.674Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892623491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122402024-02-07T18:59:55.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892781659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122412024-02-07T18:59:55.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.892839502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122422024-02-07T18:59:55.675Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:54.892938938Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122432024-02-07T18:59:55.675Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:54.892991341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122442024-02-07T18:59:55.675Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:54.893077859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122452024-02-07T18:59:55.675Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893131392Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122462024-02-07T18:59:55.675Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893166207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122472024-02-07T18:59:55.675Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893237517Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122482024-02-07T18:59:55.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893685397Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122492024-02-07T18:59:55.676Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893724692Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122502024-02-07T18:59:55.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893750818Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122512024-02-07T18:59:55.676Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.893774275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122522024-02-07T18:59:55.676Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.894357156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122532024-02-07T18:59:55.676Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.89447708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122542024-02-07T18:59:55.676Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.894923669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122552024-02-07T18:59:55.676Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.894967634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122562024-02-07T18:59:55.676Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.894986761Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122572024-02-07T18:59:55.676Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.895007808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122582024-02-07T18:59:55.677Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.895022606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122592024-02-07T18:59:55.677Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:54.895036154Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122602024-02-07T18:59:56.669Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.895018886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122612024-02-07T18:59:56.669Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.89528167Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122622024-02-07T18:59:56.675Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.896117787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122632024-02-07T18:59:56.675Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.896159641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122642024-02-07T18:59:56.675Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.896702828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122652024-02-07T18:59:56.676Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897271311Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122662024-02-07T18:59:56.676Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897300817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122672024-02-07T18:59:56.676Z{"msg":"[0]R flush_numbers[0..12]: [194, 194, 178, 191, 187, 192, 194, 172, 194, 194, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897316495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122682024-02-07T18:59:56.676Z{"msg":"[0]R generation[0..12]: [111, 111, 104, 108, 106, 109, 111, 100, 111, 111, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897330923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122692024-02-07T18:59:56.676Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897346521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122702024-02-07T18:59:56.676Z{"msg":"[1]R flush_numbers[0..12]: [194, 194, 178, 191, 187, 192, 194, 172, 194, 194, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897368648Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122712024-02-07T18:59:56.676Z{"msg":"[1]R generation[0..12]: [111, 111, 104, 108, 106, 109, 111, 100, 111, 111, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897382966Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122722024-02-07T18:59:56.676Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897403963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122732024-02-07T18:59:56.676Z{"msg":"[2]R flush_numbers[0..12]: [194, 194, 178, 191, 187, 192, 194, 172, 194, 194, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897419571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122742024-02-07T18:59:56.677Z{"msg":"[2]R generation[0..12]: [111, 111, 104, 108, 106, 109, 111, 100, 111, 111, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897440348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122752024-02-07T18:59:56.677Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897455296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122762024-02-07T18:59:56.677Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897469144Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122772024-02-07T18:59:56.677Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897490491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122782024-02-07T18:59:56.677ZWait for a query_work_queue command to finish before sending IO
122792024-02-07T18:59:56.677Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897509789Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122802024-02-07T18:59:56.677Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897524077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122812024-02-07T18:59:56.677Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897538965Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122822024-02-07T18:59:56.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897551193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122832024-02-07T18:59:56.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897564451Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122842024-02-07T18:59:56.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897596417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122852024-02-07T18:59:56.678Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897626963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122862024-02-07T18:59:56.678Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897641651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122872024-02-07T18:59:56.678Z{"msg":"2f4feab6-b542-436a-95c6-cbfe079d7974 is now active with session: 8861fa5f-081e-43fb-acb5-0116d184467c","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897674637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122882024-02-07T18:59:56.678Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897704772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583,"session_id":"8861fa5f-081e-43fb-acb5-0116d184467c"}
122892024-02-07T18:59:56.678Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-07T18:59:55.897727039Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583}
122902024-02-07T18:59:56.678ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122912024-02-07T18:59:56.678ZLoading write count information from file "/tmp/test_repair_verify.data"
122922024-02-07T18:59:56.679ZRun Repair workload
122932024-02-07T18:59:56.679Z01/30 Write block 104 len 12288 data: 11 13 12
122942024-02-07T18:59:56.683Z02/30 Read block 501 len 16384
122952024-02-07T18:59:56.683Z03/30 Write block 550 len 40960 data: 11 8 7 8 7 9 8 6 8 10
122962024-02-07T18:59:56.684Z04/30 Read block 100 len 32768
122972024-02-07T18:59:56.684Z05/30 Read block 302 len 12288
122982024-02-07T18:59:56.684Z06/30 Read block 473 len 40960
122992024-02-07T18:59:56.684Z07/30 Flush
123002024-02-07T18:59:56.684Z08/30 Write block 95 len 20480 data: 12 12 10 9 8
123012024-02-07T18:59:56.684Z09/30 Read block 187 len 16384
123022024-02-07T18:59:56.690Z10/30 Write block 401 len 32768 data: 10 11 8 7 7 9 8 8
123032024-02-07T18:59:56.690Z11/30 Read block 350 len 40960
123042024-02-07T18:59:56.690Z12/30 Write block 411 len 8192 data: 13 10
123052024-02-07T18:59:56.690Z13/30 Flush
123062024-02-07T18:59:56.691Z14/30 Write block 389 len 32768 data: 12 7 6 6 9 8 7 8
123072024-02-07T18:59:56.691Z15/30 Write block 4 len 32768 data: 6 10 9 12 13 14 10 10
123082024-02-07T18:59:56.695Z16/30 Read block 240 len 12288
123092024-02-07T18:59:56.695Z17/30 Read block 232 len 36864
123102024-02-07T18:59:56.695Z18/30 Flush
123112024-02-07T18:59:56.700Z19/30 Write block 12 len 40960 data: 10 9 10 8 10 11 9 12 13 13
123122024-02-07T18:59:56.700Z20/30 Read block 356 len 20480
123132024-02-07T18:59:56.700Z21/30 Read block 2 len 16384
123142024-02-07T18:59:56.700Z22/30 Read block 336 len 28672
123152024-02-07T18:59:56.700Z23/30 Read block 392 len 12288
123162024-02-07T18:59:56.701Z24/30 Read block 83 len 8192
123172024-02-07T18:59:56.706Z25/30 Write block 284 len 8192 data: 12 12
123182024-02-07T18:59:56.706Z26/30 Read block 418 len 16384
123192024-02-07T18:59:56.706Z27/30 Read block 538 len 24576
123202024-02-07T18:59:56.707Z28/30 Write block 317 len 16384 data: 12 11 9 8
123212024-02-07T18:59:56.707Z29/30 Write block 531 len 4096 data: 6
123222024-02-07T18:59:56.707Z30/30 Read block 156 len 24576
123232024-02-07T18:59:56.707Z----------------------------------------------------------------
123242024-02-07T18:59:56.707Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
123252024-02-07T18:59:56.707ZStates: Active Active Active
123262024-02-07T18:59:56.707ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123272024-02-07T18:59:56.707Z 1 Acked 1000 Write 3 Done Sent Done false
123282024-02-07T18:59:56.708Z 2 Acked 1001 Read 4 Done Sent Done false
123292024-02-07T18:59:56.708Z 3 Acked 1002 Write 10 Done Sent Done false
123302024-02-07T18:59:56.708Z 4 Acked 1003 Read 8 Done Sent Done false
123312024-02-07T18:59:56.708Z 5 Acked 1004 Read 3 Done Sent Done false
123322024-02-07T18:59:56.708Z 6 Acked 1005 Read 10 Done Sent Done false
123332024-02-07T18:59:56.708Z 7 Acked 1006 Flush 0 Done Sent Done false
123342024-02-07T18:59:56.708Z 8 Acked 1007 Write 5 Done Sent Done false
123352024-02-07T18:59:56.709Z 9 Acked 1008 Read 4 Done Sent Done false
123362024-02-07T18:59:56.709Z 10 Acked 1009 Write 8 Done Sent Done false
123372024-02-07T18:59:56.709Z 11 Acked 1010 Read 10 Done Sent Done false
123382024-02-07T18:59:56.709Z 12 Acked 1011 Write 2 Done Sent Done false
123392024-02-07T18:59:56.709Z 13 Acked 1012 Flush 0 Done Sent Done false
123402024-02-07T18:59:56.710Z 14 Acked 1013 Write 8 Done Sent Done false
123412024-02-07T18:59:56.710Z 15 Acked 1014 Write 8 Done Sent Done false
123422024-02-07T18:59:56.710Z 16 Acked 1015 Read 3 Done Sent Done false
123432024-02-07T18:59:56.710Z 17 Acked 1016 Read 9 Done Sent Done false
123442024-02-07T18:59:56.710Z 18 Acked 1017 Flush 0 Done Sent Done false
123452024-02-07T18:59:56.710Z 19 Acked 1018 Write 10 Done Sent Done false
123462024-02-07T18:59:56.710Z 20 Acked 1019 Read 5 Done Sent Done false
123472024-02-07T18:59:56.710Z 21 Acked 1020 Read 4 Done Sent Done false
123482024-02-07T18:59:56.711Z 22 Acked 1021 Read 7 Done Sent Done false
123492024-02-07T18:59:56.711Z 23 Acked 1022 Read 3 Done Sent Done false
123502024-02-07T18:59:56.711Z 24 Acked 1023 Read 2 Done Sent Done false
123512024-02-07T18:59:56.711Z 25 Acked 1024 Write 2 Done Sent Done false
123522024-02-07T18:59:56.711Z 26 Acked 1025 Read 4 Done Sent Done false
123532024-02-07T18:59:56.711Z 27 Acked 1026 Read 6 Done Sent Done false
123542024-02-07T18:59:56.711Z 28 Acked 1027 Write 4 Done Sent Done false
123552024-02-07T18:59:56.712Z 29 Acked 1028 Write 1 Done Sent Done false
123562024-02-07T18:59:56.712Z 30 Acked 1029 Read 6 Done Sent Done false
123572024-02-07T18:59:56.712Z STATES DS:0 DS:1 DS:2 TOTAL
123582024-02-07T18:59:56.712Z New 0 0 0 0
123592024-02-07T18:59:56.712Z Sent 0 30 0 30
123602024-02-07T18:59:56.712Z Done 30 0 30 60
123612024-02-07T18:59:56.712Z Skipped 0 0 0 0
123622024-02-07T18:59:56.713Z Error 0 0 0 0
123632024-02-07T18:59:56.713ZLast Flush: 1017 0 1017
123642024-02-07T18:59:56.713ZDownstairs last five completed:
123652024-02-07T18:59:56.713ZUpstairs last five completed: 30 29 28 27 26
123662024-02-07T18:59:56.713Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T18:59:55.930765441Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1583}
123672024-02-07T18:59:56.713ZWrote out file "/tmp/test_repair_verify.data"
123682024-02-07T18:59:56.718Z
123692024-02-07T18:59:56.718ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123702024-02-07T18:59:56.731ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123712024-02-07T18:59:56.732Z 0 000-019 112 111 112 197 194 197 T F T <---
123722024-02-07T18:59:56.732Z 1 020-039 111 111 111 194 194 194 T F T <---
123732024-02-07T18:59:56.732Z 2 040-059 104 104 104 178 178 178 F F F
123742024-02-07T18:59:56.732Z 3 060-079 108 108 108 191 191 191 F F F
123752024-02-07T18:59:56.732Z 4 080-099 112 106 112 196 187 196 F F F <---
123762024-02-07T18:59:56.732Z 5 100-119 112 109 112 195 192 195 F F F <---
123772024-02-07T18:59:56.732Z 6 120-139 111 111 111 194 194 194 F F F
123782024-02-07T18:59:56.732Z 7 140-159 100 100 100 172 172 172 F F F
123792024-02-07T18:59:56.733Z 8 160-179 111 111 111 194 194 194 F F F
123802024-02-07T18:59:56.733Z 9 180-199 111 111 111 194 194 194 F F F
123812024-02-07T18:59:56.733Z 10 200-219 106 106 106 187 187 187 F F F
123822024-02-07T18:59:56.733Z 11 220-239 108 108 108 190 190 190 F F F
123832024-02-07T18:59:56.748Z 12 240-259 104 104 104 180 180 180 F F F
123842024-02-07T18:59:56.748Z 13 260-279 111 111 111 194 194 194 F F F
123852024-02-07T18:59:56.748Z 14 280-299 108 108 108 191 191 191 T F T <---
123862024-02-07T18:59:56.749Z 15 300-319 107 107 107 188 188 188 T F T <---
123872024-02-07T18:59:56.749Z 16 320-339 106 106 106 186 186 186 T F T <---
123882024-02-07T18:59:56.749Z 17 340-359 108 108 108 191 191 191 F F F
123892024-02-07T18:59:56.749Z 18 360-379 108 108 108 190 190 190 F F F
123902024-02-07T18:59:56.749Z 19 380-399 112 109 112 197 192 197 F F F <---
123912024-02-07T18:59:56.749Z 20 400-419 112 105 112 196 184 196 F F F <---
123922024-02-07T18:59:56.749Z 21 420-439 111 111 111 194 194 194 F F F
123932024-02-07T18:59:56.749Z 22 440-459 110 110 110 193 193 193 F F F
123942024-02-07T18:59:56.749Z 23 460-479 106 106 106 186 186 186 F F F
123952024-02-07T18:59:56.749Z 24 480-499 108 108 108 191 191 191 F F F
123962024-02-07T18:59:56.749Z 25 500-519 108 108 108 191 191 191 F F F
123972024-02-07T18:59:56.749Z 26 520-539 110 110 110 193 193 193 T F T <---
123982024-02-07T18:59:56.750Z 27 540-559 112 106 112 195 187 195 F F F <---
123992024-02-07T18:59:56.750Z 28 560-579 110 110 110 193 193 193 F F F
124002024-02-07T18:59:56.750Z 29 580-599 108 108 108 190 190 190 F F F
124012024-02-07T18:59:56.750ZMax gen: 112, Max flush: 197
124022024-02-07T18:59:56.750ZError: Difference in extent metadata found!
124032024-02-07T18:59:56.750ZA Difference in extent metadata is expected here
124042024-02-07T18:59:56.750ZOn loop 56
124052024-02-07T18:59:56.750Z
124062024-02-07T18:59:56.750ZVerifying data now
124072024-02-07T19:00:01.291ZNo change after verify
124082024-02-07T19:00:01.291ZLoop: 56 Downstairs dump after verify (and repair):
124092024-02-07T19:00:01.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124102024-02-07T19:00:01.307Z 0 000-019 113 113 113 198 198 198 F F F
124112024-02-07T19:00:01.307Z 1 020-039 113 113 113 198 198 198 F F F
124122024-02-07T19:00:01.307Z 2 040-059 104 104 104 178 178 178 F F F
124132024-02-07T19:00:01.307Z 3 060-079 108 108 108 191 191 191 F F F
124142024-02-07T19:00:01.307Z 4 080-099 112 112 112 196 196 196 F F F
124152024-02-07T19:00:01.307Z 5 100-119 112 112 112 195 195 195 F F F
124162024-02-07T19:00:01.307Z 6 120-139 111 111 111 194 194 194 F F F
124172024-02-07T19:00:01.307Z 7 140-159 100 100 100 172 172 172 F F F
124182024-02-07T19:00:01.308Z 8 160-179 111 111 111 194 194 194 F F F
124192024-02-07T19:00:01.308Z 9 180-199 111 111 111 194 194 194 F F F
124202024-02-07T19:00:01.308Z 10 200-219 106 106 106 187 187 187 F F F
124212024-02-07T19:00:01.308Z 11 220-239 108 108 108 190 190 190 F F F
124222024-02-07T19:00:01.308Z 12 240-259 104 104 104 180 180 180 F F F
124232024-02-07T19:00:01.308Z 13 260-279 111 111 111 194 194 194 F F F
124242024-02-07T19:00:01.308Z 14 280-299 113 113 113 198 198 198 F F F
124252024-02-07T19:00:01.313Z 15 300-319 113 113 113 198 198 198 F F F
124262024-02-07T19:00:01.314Z 16 320-339 113 113 113 198 198 198 F F F
124272024-02-07T19:00:01.314Z 17 340-359 108 108 108 191 191 191 F F F
124282024-02-07T19:00:01.314Z 18 360-379 108 108 108 190 190 190 F F F
124292024-02-07T19:00:01.314Z 19 380-399 112 112 112 197 197 197 F F F
124302024-02-07T19:00:01.314Z 20 400-419 112 112 112 196 196 196 F F F
124312024-02-07T19:00:01.314Z 21 420-439 111 111 111 194 194 194 F F F
124322024-02-07T19:00:01.315Z 22 440-459 110 110 110 193 193 193 F F F
124332024-02-07T19:00:01.315Z 23 460-479 106 106 106 186 186 186 F F F
124342024-02-07T19:00:01.315Z 24 480-499 108 108 108 191 191 191 F F F
124352024-02-07T19:00:01.315Z 25 500-519 108 108 108 191 191 191 F F F
124362024-02-07T19:00:01.315Z 26 520-539 113 113 113 198 198 198 F F F
124372024-02-07T19:00:01.315Z 27 540-559 112 112 112 195 195 195 F F F
124382024-02-07T19:00:01.315Z 28 560-579 110 110 110 193 193 193 F F F
124392024-02-07T19:00:01.315Z 29 580-599 108 108 108 190 190 190 F F F
124402024-02-07T19:00:01.316ZMax gen: 113, Max flush: 198
124412024-02-07T19:00:01.316Z
124422024-02-07T19:00:01.316ZBegin loop 57
124432024-02-07T19:00:01.316ZDownstairs to restart: 0
124442024-02-07T19:00:01.740Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554255622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591}
124452024-02-07T19:00:01.740Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554524205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591}
124462024-02-07T19:00:01.747Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554570469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591}
124472024-02-07T19:00:01.747ZCrucible runtime is spawned
124482024-02-07T19:00:01.747Z{"msg":"Crucible 235c554c-97f6-43c4-8f35-ce951ddc2206 has session id: c725e563-f0d4-48dd-8fca-d1d166399990","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554632541Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124492024-02-07T19:00:01.748Z{"msg":"Upstairs opts: Upstairs UUID: 235c554c-97f6-43c4-8f35-ce951ddc2206, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554686243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124502024-02-07T19:00:01.748Z{"msg":"Crucible stats registered with UUID: 235c554c-97f6-43c4-8f35-ce951ddc2206","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.55471372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124512024-02-07T19:00:01.748Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554753125Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591}
124522024-02-07T19:00:01.748Z{"msg":"235c554c-97f6-43c4-8f35-ce951ddc2206 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554769182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124532024-02-07T19:00:01.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554794379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124542024-02-07T19:00:01.748Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554813126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124552024-02-07T19:00:01.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554849872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124562024-02-07T19:00:01.748Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554904594Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124572024-02-07T19:00:01.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.554942289Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124582024-02-07T19:00:01.748Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555013989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124592024-02-07T19:00:01.748Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555056293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124602024-02-07T19:00:01.748Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555101178Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124612024-02-07T19:00:01.749Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555143182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124622024-02-07T19:00:01.749Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555169958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124632024-02-07T19:00:01.749Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:00.555200074Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124642024-02-07T19:00:01.749Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:00.55522981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124652024-02-07T19:00:01.749Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:00.555285242Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124662024-02-07T19:00:01.749Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555337915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124672024-02-07T19:00:01.749Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.55537805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124682024-02-07T19:00:01.749Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555405356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124692024-02-07T19:00:01.749Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555442481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124702024-02-07T19:00:01.749Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.55582083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124712024-02-07T19:00:01.750Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555853486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124722024-02-07T19:00:01.750Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555870014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124732024-02-07T19:00:01.750Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.555886681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124742024-02-07T19:00:01.750Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.556377325Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124752024-02-07T19:00:01.750Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.556525775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124762024-02-07T19:00:01.750Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.556884256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124772024-02-07T19:00:01.750Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.55693475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124782024-02-07T19:00:01.750Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.556963126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124792024-02-07T19:00:01.750Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.557063582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124802024-02-07T19:00:01.750Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.557103587Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124812024-02-07T19:00:01.750Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:00.557120754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124822024-02-07T19:00:02.331Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.556001905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124832024-02-07T19:00:02.331Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.556170742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124842024-02-07T19:00:02.340Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.557080069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124852024-02-07T19:00:02.340Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.557118534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124862024-02-07T19:00:02.340Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.55766415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124872024-02-07T19:00:02.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558305773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124882024-02-07T19:00:02.340Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558347538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124892024-02-07T19:00:02.340Z{"msg":"[0]R flush_numbers[0..12]: [198, 198, 178, 191, 196, 195, 194, 172, 194, 194, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558368445Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124902024-02-07T19:00:02.340Z{"msg":"[0]R generation[0..12]: [113, 113, 104, 108, 112, 112, 111, 100, 111, 111, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558388932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124912024-02-07T19:00:02.341Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558417278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124922024-02-07T19:00:02.341Z{"msg":"[1]R flush_numbers[0..12]: [198, 198, 178, 191, 196, 195, 194, 172, 194, 194, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558436806Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124932024-02-07T19:00:02.341Z{"msg":"[1]R generation[0..12]: [113, 113, 104, 108, 112, 112, 111, 100, 111, 111, 106, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558454383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124942024-02-07T19:00:02.341Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558482259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124952024-02-07T19:00:02.341Z{"msg":"[2]R flush_numbers[0..12]: [198, 198, 178, 191, 196, 195, 194, 172, 194, 194, 187, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558507016Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124962024-02-07T19:00:02.341Z{"msg":"[2]R generation[0..12]: [113, 113, 104, 108, 112, 112, 111, 100, 111, 111, 106, 108]","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO
124972024-02-07T19:00:02.341Z":30,"time":"2024-02-07T19:00:01.55855352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124982024-02-07T19:00:02.341Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558601143Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
124992024-02-07T19:00:02.341Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558633219Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125002024-02-07T19:00:02.341Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558651207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125012024-02-07T19:00:02.341Z{"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558681963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125022024-02-07T19:00:02.341Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558713828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125032024-02-07T19:00:02.341Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558758382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125042024-02-07T19:00:02.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558802736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125052024-02-07T19:00:02.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558841751Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125062024-02-07T19:00:02.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558862558Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125072024-02-07T19:00:02.342Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558904113Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125082024-02-07T19:00:02.342Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558947867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125092024-02-07T19:00:02.342Z{"msg":"235c554c-97f6-43c4-8f35-ce951ddc2206 is now active with session: c725e563-f0d4-48dd-8fca-d1d166399990","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.558977863Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125102024-02-07T19:00:02.342Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.55899732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591,"session_id":"c725e563-f0d4-48dd-8fca-d1d166399990"}
125112024-02-07T19:00:02.342Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:01.559038984Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591}
125122024-02-07T19:00:02.342ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125132024-02-07T19:00:02.342ZLoading write count information from file "/tmp/test_repair_verify.data"
125142024-02-07T19:00:02.342ZRun Repair workload
125152024-02-07T19:00:02.342Z01/30 Flush
125162024-02-07T19:00:02.342Z02/30 Write block 163 len 36864 data: 10 11 11 8 8 11 11 10 10
125172024-02-07T19:00:02.348Z03/30 Read block 88 len 24576
125182024-02-07T19:00:02.348Z04/30 Write block 312 len 12288 data: 6 9 9
125192024-02-07T19:00:02.348Z05/30 Write block 142 len 40960 data: 10 11 11 11 13 14 13 16 10 10
125202024-02-07T19:00:02.348Z06/30 Read block 71 len 32768
125212024-02-07T19:00:02.348Z07/30 Read block 435 len 24576
125222024-02-07T19:00:02.348Z08/30 Flush
125232024-02-07T19:00:02.348Z09/30 Write block 92 len 8192 data: 9 9
125242024-02-07T19:00:02.348Z10/30 Read block 211 len 8192
125252024-02-07T19:00:02.348Z11/30 Read block 545 len 16384
125262024-02-07T19:00:02.348Z12/30 Read block 328 len 8192
125272024-02-07T19:00:02.348Z13/30 Read block 350 len 16384
125282024-02-07T19:00:02.352Z14/30 Flush
125292024-02-07T19:00:02.353Z15/30 Write block 264 len 12288 data: 12 10 11
125302024-02-07T19:00:02.353Z16/30 Write block 298 len 24576 data: 9 6 6 6 7 4
125312024-02-07T19:00:02.353Z17/30 Write block 187 len 16384 data: 5 6 6 5
125322024-02-07T19:00:02.353Z18/30 Read block 332 len 16384
125332024-02-07T19:00:02.359Z19/30 Read block 572 len 20480
125342024-02-07T19:00:02.360Z20/30 Flush
125352024-02-07T19:00:02.360Z21/30 Write block 148 len 36864 data: 14 17 11 11 10 11 11 7 6
125362024-02-07T19:00:02.360Z22/30 Write block 398 len 20480 data: 9 11 12 11 12
125372024-02-07T19:00:02.360Z23/30 Read block 152 len 20480
125382024-02-07T19:00:02.360Z24/30 Read block 44 len 20480
125392024-02-07T19:00:02.364Z25/30 Write block 124 len 28672 data: 10 10 10 10 11 11 9
125402024-02-07T19:00:02.364Z26/30 Write block 231 len 36864 data: 9 12 12 12 17 16 17 16 16
125412024-02-07T19:00:02.364Z27/30 Read block 280 len 4096
125422024-02-07T19:00:02.364Z28/30 Write block 17 len 28672 data: 12 10 13 14 14 13 13
125432024-02-07T19:00:02.365Z29/30 Write block 330 len 8192 data: 7 10
125442024-02-07T19:00:02.365Z30/30 Read block 274 len 24576
125452024-02-07T19:00:02.376Z----------------------------------------------------------------
125462024-02-07T19:00:02.376Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:29
125472024-02-07T19:00:02.376ZStates: Active Active Active
125482024-02-07T19:00:02.376ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125492024-02-07T19:00:02.376Z 2 Acked 1001 Write 9 Sent Done Done false
125502024-02-07T19:00:02.376Z 3 Acked 1002 Read 6 Sent Done Done false
125512024-02-07T19:00:02.376Z 4 Acked 1003 Write 3 Sent Done Done false
125522024-02-07T19:00:02.376Z 5 Acked 1004 Write 10 Sent Done Done false
125532024-02-07T19:00:02.376Z 6 Acked 1005 Read 8 Sent Done Done false
125542024-02-07T19:00:02.376Z 7 Acked 1006 Read 6 Sent Done Done false
125552024-02-07T19:00:02.377Z 8 Acked 1007 Flush 0 Sent Done Done false
125562024-02-07T19:00:02.377Z 9 Acked 1008 Write 2 Sent Done Done false
125572024-02-07T19:00:02.377Z 10 Acked 1009 Read 2 Sent Done Done false
125582024-02-07T19:00:02.377Z 11 Acked 1010 Read 4 Sent Done Done false
125592024-02-07T19:00:02.377Z 12 Acked 1011 Read 2 Sent Done Done false
125602024-02-07T19:00:02.377Z 13 Acked 1012 Read 4 Sent Done Done false
125612024-02-07T19:00:02.377Z 14 Acked 1013 Flush 0 Sent Done Done false
125622024-02-07T19:00:02.377Z 15 Acked 1014 Write 3 Sent Done Done false
125632024-02-07T19:00:02.377Z 16 Acked 1015 Write 6 Sent Done Done false
125642024-02-07T19:00:02.377Z 17 Acked 1016 Write 4 Sent Done Done false
125652024-02-07T19:00:02.377Z 18 Acked 1017 Read 4 Sent Done Done false
125662024-02-07T19:00:02.377Z 19 Acked 1018 Read 5 Sent Done Done false
125672024-02-07T19:00:02.377Z 20 Acked 1019 Flush 0 Sent Done Done false
125682024-02-07T19:00:02.378Z 21 Acked 1020 Write 9 Sent Done Done false
125692024-02-07T19:00:02.378Z 22 Acked 1021 Write 5 Sent Done Done false
125702024-02-07T19:00:02.378Z 23 Acked 1022 Read 5 Sent Done Done false
125712024-02-07T19:00:02.378Z 24 Acked 1023 Read 5 Sent Done Done false
125722024-02-07T19:00:02.378Z 25 Acked 1024 Write 7 Sent Done Done false
125732024-02-07T19:00:02.378Z 26 Acked 1025 Write 9 Sent Done Done false
125742024-02-07T19:00:02.378Z 27 Acked 1026 Read 1 Sent Done Done false
125752024-02-07T19:00:02.378Z 28 Acked 1027 Write 7 Sent Done Done false
125762024-02-07T19:00:02.378Z 29 Acked 1028 Write 2 Sent Done Done false
125772024-02-07T19:00:02.378Z 30 Acked 1029 Read 6 Sent Done Done false
125782024-02-07T19:00:02.378Z STATES DS:0 DS:1 DS:2 TOTAL
125792024-02-07T19:00:02.378Z New 0 0 0 0
125802024-02-07T19:00:02.378Z Sent 29 0 0 29
125812024-02-07T19:00:02.379Z Done 0 29 29 58
125822024-02-07T19:00:02.379Z Skipped 0 0 0 0
125832024-02-07T19:00:02.379Z Error 0 0 0 0
125842024-02-07T19:00:02.379ZLast Flush: 1000 1019 1019
125852024-02-07T19:00:02.379ZDownstairs last five completed: 1000
125862024-02-07T19:00:02.379ZUpstairs last five completed: 30 29 28 27 26
125872024-02-07T19:00:02.379Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:01.590940329Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1591}
125882024-02-07T19:00:02.379ZWrote out file "/tmp/test_repair_verify.data"
125892024-02-07T19:00:02.379Z
125902024-02-07T19:00:02.384ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125912024-02-07T19:00:02.393ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125922024-02-07T19:00:02.393Z 0 000-019 113 113 113 198 198 198 F T T <---
125932024-02-07T19:00:02.393Z 1 020-039 113 113 113 198 198 198 F T T <---
125942024-02-07T19:00:02.393Z 2 040-059 104 104 104 178 178 178 F F F
125952024-02-07T19:00:02.394Z 3 060-079 108 108 108 191 191 191 F F F
125962024-02-07T19:00:02.394Z 4 080-099 112 114 114 196 201 201 F F F <---
125972024-02-07T19:00:02.394Z 5 100-119 112 112 112 195 195 195 F F F
125982024-02-07T19:00:02.394Z 6 120-139 111 111 111 194 194 194 F T T <---
125992024-02-07T19:00:02.394Z 7 140-159 100 114 114 172 200 200 F T T <---
126002024-02-07T19:00:02.394Z 8 160-179 111 114 114 194 200 200 F F F <---
126012024-02-07T19:00:02.394Z 9 180-199 111 114 114 194 202 202 F F F <---
126022024-02-07T19:00:02.394Z 10 200-219 106 106 106 187 187 187 F F F
126032024-02-07T19:00:02.394Z 11 220-239 108 108 108 190 190 190 F T T <---
126042024-02-07T19:00:02.394Z 12 240-259 104 104 104 180 180 180 F F F
126052024-02-07T19:00:02.400Z 13 260-279 111 114 114 194 202 202 F F F <---
126062024-02-07T19:00:02.400Z 14 280-299 113 114 114 198 202 202 F F F <---
126072024-02-07T19:00:02.400Z 15 300-319 113 114 114 198 202 202 F F F <---
126082024-02-07T19:00:02.400Z 16 320-339 113 113 113 198 198 198 F T T <---
126092024-02-07T19:00:02.400Z 17 340-359 108 108 108 191 191 191 F F F
126102024-02-07T19:00:02.400Z 18 360-379 108 108 108 190 190 190 F F F
126112024-02-07T19:00:02.400Z 19 380-399 112 112 112 197 197 197 F T T <---
126122024-02-07T19:00:02.401Z 20 400-419 112 112 112 196 196 196 F T T <---
126132024-02-07T19:00:02.401Z 21 420-439 111 111 111 194 194 194 F F F
126142024-02-07T19:00:02.401Z 22 440-459 110 110 110 193 193 193 F F F
126152024-02-07T19:00:02.401Z 23 460-479 106 106 106 186 186 186 F F F
126162024-02-07T19:00:02.401Z 24 480-499 108 108 108 191 191 191 F F F
126172024-02-07T19:00:02.401Z 25 500-519 108 108 108 191 191 191 F F F
126182024-02-07T19:00:02.401Z 26 520-539 113 113 113 198 198 198 F F F
126192024-02-07T19:00:02.401Z 27 540-559 112 112 112 195 195 195 F F F
126202024-02-07T19:00:02.401Z 28 560-579 110 110 110 193 193 193 F F F
126212024-02-07T19:00:02.401Z 29 580-599 108 108 108 190 190 190 F F F
126222024-02-07T19:00:02.401ZMax gen: 114, Max flush: 202
126232024-02-07T19:00:02.401ZError: Difference in extent metadata found!
126242024-02-07T19:00:02.402ZA Difference in extent metadata is expected here
126252024-02-07T19:00:02.402ZOn loop 57
126262024-02-07T19:00:02.402Z
126272024-02-07T19:00:02.406ZVerifying data now
126282024-02-07T19:00:07.076ZNo change after verify
126292024-02-07T19:00:07.077ZLoop: 57 Downstairs dump after verify (and repair):
126302024-02-07T19:00:07.092ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126312024-02-07T19:00:07.092Z 0 000-019 115 115 115 203 203 203 F F F
126322024-02-07T19:00:07.092Z 1 020-039 115 115 115 203 203 203 F F F
126332024-02-07T19:00:07.092Z 2 040-059 104 104 104 178 178 178 F F F
126342024-02-07T19:00:07.092Z 3 060-079 108 108 108 191 191 191 F F F
126352024-02-07T19:00:07.092Z 4 080-099 114 114 114 201 201 201 F F F
126362024-02-07T19:00:07.093Z 5 100-119 112 112 112 195 195 195 F F F
126372024-02-07T19:00:07.093Z 6 120-139 115 115 115 203 203 203 F F F
126382024-02-07T19:00:07.097Z 7 140-159 115 115 115 203 203 203 F F F
126392024-02-07T19:00:07.097Z 8 160-179 114 114 114 200 200 200 F F F
126402024-02-07T19:00:07.097Z 9 180-199 114 114 114 202 202 202 F F F
126412024-02-07T19:00:07.097Z 10 200-219 106 106 106 187 187 187 F F F
126422024-02-07T19:00:07.097Z 11 220-239 115 115 115 203 203 203 F F F
126432024-02-07T19:00:07.097Z 12 240-259 104 104 104 180 180 180 F F F
126442024-02-07T19:00:07.098Z 13 260-279 114 114 114 202 202 202 F F F
126452024-02-07T19:00:07.098Z 14 280-299 114 114 114 202 202 202 F F F
126462024-02-07T19:00:07.098Z 15 300-319 114 114 114 202 202 202 F F F
126472024-02-07T19:00:07.098Z 16 320-339 115 115 115 203 203 203 F F F
126482024-02-07T19:00:07.098Z 17 340-359 108 108 108 191 191 191 F F F
126492024-02-07T19:00:07.098Z 18 360-379 108 108 108 190 190 190 F F F
126502024-02-07T19:00:07.098Z 19 380-399 115 115 115 203 203 203 F F F
126512024-02-07T19:00:07.098Z 20 400-419 115 115 115 203 203 203 F F F
126522024-02-07T19:00:07.099Z 21 420-439 111 111 111 194 194 194 F F F
126532024-02-07T19:00:07.099Z 22 440-459 110 110 110 193 193 193 F F F
126542024-02-07T19:00:07.099Z 23 460-479 106 106 106 186 186 186 F F F
126552024-02-07T19:00:07.099Z 24 480-499 108 108 108 191 191 191 F F F
126562024-02-07T19:00:07.099Z 25 500-519 108 108 108 191 191 191 F F F
126572024-02-07T19:00:07.099Z 26 520-539 113 113 113 198 198 198 F F F
126582024-02-07T19:00:07.099Z 27 540-559 112 112 112 195 195 195 F F F
126592024-02-07T19:00:07.100Z 28 560-579 110 110 110 193 193 193 F F F
126602024-02-07T19:00:07.100Z 29 580-599 108 108 108 190 190 190 F F F
126612024-02-07T19:00:07.100ZMax gen: 115, Max flush: 203
126622024-02-07T19:00:07.100Z
126632024-02-07T19:00:07.100ZBegin loop 58
126642024-02-07T19:00:07.100ZDownstairs to restart: 1
126652024-02-07T19:00:07.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338082997Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599}
126662024-02-07T19:00:07.128Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338349911Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599}
126672024-02-07T19:00:07.136Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338375387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599}
126682024-02-07T19:00:07.136ZCrucible runtime is spawned
126692024-02-07T19:00:07.136Z{"msg":"Crucible 41fc9ba7-c69d-428a-a507-3ce382466b4e has session id: a7af150f-ef29-457b-8f62-be25c5c39e73","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338455386Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126702024-02-07T19:00:07.136Z{"msg":"Upstairs opts: Upstairs UUID: 41fc9ba7-c69d-428a-a507-3ce382466b4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338535955Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126712024-02-07T19:00:07.136Z{"msg":"Crucible stats registered with UUID: 41fc9ba7-c69d-428a-a507-3ce382466b4e","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338573281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126722024-02-07T19:00:07.136Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338621694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599}
126732024-02-07T19:00:07.136Z{"msg":"41fc9ba7-c69d-428a-a507-3ce382466b4e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.33865405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126742024-02-07T19:00:07.136Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338696594Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126752024-02-07T19:00:07.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338730949Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126762024-02-07T19:00:07.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338762495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126772024-02-07T19:00:07.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338813038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126782024-02-07T19:00:07.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338843554Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126792024-02-07T19:00:07.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.33887277Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126802024-02-07T19:00:07.137Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338902146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126812024-02-07T19:00:07.137Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.33894661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126822024-02-07T19:00:07.137Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.338980166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126832024-02-07T19:00:07.137Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:06.339028679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126842024-02-07T19:00:07.137Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:06.339066134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126852024-02-07T19:00:07.138Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:06.339102519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126862024-02-07T19:00:07.138Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339128695Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126872024-02-07T19:00:07.138Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339156492Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126882024-02-07T19:00:07.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339183438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126892024-02-07T19:00:07.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339224422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126902024-02-07T19:00:07.138Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339265917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126912024-02-07T19:00:07.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339654284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126922024-02-07T19:00:07.138Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339707527Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126932024-02-07T19:00:07.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339732974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126942024-02-07T19:00:07.139Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.339768009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126952024-02-07T19:00:07.139Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340260212Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126962024-02-07T19:00:07.139Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340391595Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126972024-02-07T19:00:07.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340822736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126982024-02-07T19:00:07.139Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340865171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
126992024-02-07T19:00:07.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340891907Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127002024-02-07T19:00:07.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340948169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127012024-02-07T19:00:07.139Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.340985514Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127022024-02-07T19:00:07.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:06.341001372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127032024-02-07T19:00:08.115Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.339486996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127042024-02-07T19:00:08.122Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.339613709Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127052024-02-07T19:00:08.122Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.340583248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127062024-02-07T19:00:08.123Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.340624312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127072024-02-07T19:00:08.123Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341123165Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127082024-02-07T19:00:08.123Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341700157Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127092024-02-07T19:00:08.123Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.34175193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127102024-02-07T19:00:08.123Z{"msg":"[0]R flush_numbers[0..12]: [203, 203, 178, 191, 201, 195, 203, 203, 200, 202, 187, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341787525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127112024-02-07T19:00:08.123Z{"msg":"[0]R generation[0..12]: [115, 115, 104, 108, 114, 112, 115, 115, 114, 114, 106, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.34182516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127122024-02-07T19:00:08.124Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341845387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127132024-02-07T19:00:08.124Z{"msg":"[1]R flush_numbers[0..12]: [203, 203, 178, 191, 201, 195, 203, 203, 200, 202, 187, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341872253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127142024-02-07T19:00:08.124Z{"msg":"[1]R generation[0..12]: [115, 115, 104, 108, 114, 112, 115, 115, 114, 114, 106, 115]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
127152024-02-07T19:00:08.124Z,"time":"2024-02-07T19:00:07.341891211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127162024-02-07T19:00:08.124Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341926296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127172024-02-07T19:00:08.124Z{"msg":"[2]R flush_numbers[0..12]: [203, 203, 178, 191, 201, 195, 203, 203, 200, 202, 187, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341952713Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127182024-02-07T19:00:08.124Z{"msg":"[2]R generation[0..12]: [115, 115, 104, 108, 114, 112, 115, 115, 114, 114, 106, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.341980209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127192024-02-07T19:00:08.125Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342005316Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127202024-02-07T19:00:08.125Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342033732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127212024-02-07T19:00:08.125Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342052309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127222024-02-07T19:00:08.125Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342079296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127232024-02-07T19:00:08.125Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342101373Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127242024-02-07T19:00:08.125Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.34212318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127252024-02-07T19:00:08.125Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342162654Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127262024-02-07T19:00:08.125Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342209758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127272024-02-07T19:00:08.126Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342254932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127282024-02-07T19:00:08.126Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342295096Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127292024-02-07T19:00:08.126Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342335111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127302024-02-07T19:00:08.126Z{"msg":"41fc9ba7-c69d-428a-a507-3ce382466b4e is now active with session: a7af150f-ef29-457b-8f62-be25c5c39e73","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342371066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127312024-02-07T19:00:08.126Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127322024-02-07T19:00:08.126Z,"time":"2024-02-07T19:00:07.342443846Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599,"session_id":"a7af150f-ef29-457b-8f62-be25c5c39e73"}
127332024-02-07T19:00:08.126Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:07.342464764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599}
127342024-02-07T19:00:08.126ZLoading write count information from file "/tmp/test_repair_verify.data"
127352024-02-07T19:00:08.126ZRun Repair workload
127362024-02-07T19:00:08.126Z01/30 Flush
127372024-02-07T19:00:08.127Z02/30 Write block 213 len 24576 data: 10 8 5 4 4 5
127382024-02-07T19:00:08.131Z03/30 Write block 522 len 20480 data: 4 4 7 6 7
127392024-02-07T19:00:08.132Z04/30 Write block 260 len 32768 data: 8 7 9 12 13 11 12 12
127402024-02-07T19:00:08.132Z05/30 Read block 246 len 12288
127412024-02-07T19:00:08.132Z06/30 Read block 226 len 12288
127422024-02-07T19:00:08.132Z07/30 Read block 383 len 36864
127432024-02-07T19:00:08.132Z08/30 Read block 89 len 40960
127442024-02-07T19:00:08.132Z09/30 Write block 29 len 40960 data: 12 12 13 11 11 12 11 11 8 6
127452024-02-07T19:00:08.132Z10/30 Write block 506 len 28672 data: 11 9 9 9 11 8 11
127462024-02-07T19:00:08.133Z11/30 Read block 62 len 16384
127472024-02-07T19:00:08.133Z12/30 Flush
127482024-02-07T19:00:08.137Z13/30 Write block 29 len 32768 data: 13 13 14 12 12 13 12 12
127492024-02-07T19:00:08.137Z14/30 Read block 53 len 4096
127502024-02-07T19:00:08.137Z15/30 Write block 164 len 36864 data: 12 12 9 9 12 12 11 11 10
127512024-02-07T19:00:08.137Z16/30 Read block 270 len 24576
127522024-02-07T19:00:08.138Z17/30 Write block 201 len 16384 data: 7 10 8 10
127532024-02-07T19:00:08.138Z18/30 Write block 341 len 32768 data: 7 7 7 8 12 14 14 13
127542024-02-07T19:00:08.138Z19/30 Write block 111 len 8192 data: 11 8
127552024-02-07T19:00:08.143Z20/30 Write block 183 len 40960 data: 4 3 4 5 6 7 7 6 10 11
127562024-02-07T19:00:08.143Z21/30 Write block 439 len 40960 data: 9 10 10 11 12 12 12 10 10 10
127572024-02-07T19:00:08.144Z22/30 Read block 554 len 32768
127582024-02-07T19:00:08.144Z23/30 Read block 85 len 4096
127592024-02-07T19:00:08.144Z24/30 Write block 153 len 32768 data: 12 12 8 7 5 7 6 6
127602024-02-07T19:00:08.149Z25/30 Read block 547 len 28672
127612024-02-07T19:00:08.149Z26/30 Read block 548 len 8192
127622024-02-07T19:00:08.149Z27/30 Read block 585 len 12288
127632024-02-07T19:00:08.149Z28/30 Read block 100 len 40960
127642024-02-07T19:00:08.149Z29/30 Read block 150 len 32768
127652024-02-07T19:00:08.150Z30/30 Read block 476 len 20480
127662024-02-07T19:00:08.161Z----------------------------------------------------------------
127672024-02-07T19:00:08.161Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:29
127682024-02-07T19:00:08.162ZStates: Active Active Active
127692024-02-07T19:00:08.162ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127702024-02-07T19:00:08.162Z 2 Acked 1001 Write 6 Done Sent Done false
127712024-02-07T19:00:08.162Z 3 Acked 1002 Write 5 Done Sent Done false
127722024-02-07T19:00:08.162Z 4 Acked 1003 Write 8 Done Sent Done false
127732024-02-07T19:00:08.162Z 5 Acked 1004 Read 3 Done Sent Done false
127742024-02-07T19:00:08.162Z 6 Acked 1005 Read 3 Done Sent Done false
127752024-02-07T19:00:08.163Z 7 Acked 1006 Read 9 Done Sent Done false
127762024-02-07T19:00:08.163Z 8 Acked 1007 Read 10 Done Sent Done false
127772024-02-07T19:00:08.163Z 9 Acked 1008 Write 10 Done Sent Done false
127782024-02-07T19:00:08.163Z 10 Acked 1009 Write 7 Done Sent Done false
127792024-02-07T19:00:08.164Z 11 Acked 1010 Read 4 Done Sent Done false
127802024-02-07T19:00:08.164Z 12 Acked 1011 Flush 0 Done Sent Done false
127812024-02-07T19:00:08.164Z 13 Acked 1012 Write 8 Done Sent Done false
127822024-02-07T19:00:08.164Z 14 Acked 1013 Read 1 Done Sent Done false
127832024-02-07T19:00:08.164Z 15 Acked 1014 Write 9 Done Sent Done false
127842024-02-07T19:00:08.165Z 16 Acked 1015 Read 6 Done Sent Done false
127852024-02-07T19:00:08.165Z 17 Acked 1016 Write 4 Done Sent Done false
127862024-02-07T19:00:08.165Z 18 Acked 1017 Write 8 Done Sent Done false
127872024-02-07T19:00:08.165Z 19 Acked 1018 Write 2 Done Sent Done false
127882024-02-07T19:00:08.165Z 20 Acked 1019 Write 10 Done Sent Done false
127892024-02-07T19:00:08.165Z 21 Acked 1020 Write 10 Done Sent Done false
127902024-02-07T19:00:08.166Z 22 Acked 1021 Read 8 Done Sent Done false
127912024-02-07T19:00:08.166Z 23 Acked 1022 Read 1 Done Sent Done false
127922024-02-07T19:00:08.166Z 24 Acked 1023 Write 8 Done Sent Done false
127932024-02-07T19:00:08.166Z 25 Acked 1024 Read 7 Done Sent Done false
127942024-02-07T19:00:08.166Z 26 Acked 1025 Read 2 Done Sent Done false
127952024-02-07T19:00:08.166Z 27 Acked 1026 Read 3 Done Sent Done false
127962024-02-07T19:00:08.166Z 28 Acked 1027 Read 10 Done Sent Done false
127972024-02-07T19:00:08.166Z 29 Acked 1028 Read 8 Done Sent Done false
127982024-02-07T19:00:08.167Z 30 Acked 1029 Read 5 Done Sent Done false
127992024-02-07T19:00:08.167Z STATES DS:0 DS:1 DS:2 TOTAL
128002024-02-07T19:00:08.167Z New 0 0 0 0
128012024-02-07T19:00:08.167Z Sent 0 29 0 29
128022024-02-07T19:00:08.167Z Done 29 0 29 58
128032024-02-07T19:00:08.167Z Skipped 0 0 0 0
128042024-02-07T19:00:08.167Z Error 0 0 0 0
128052024-02-07T19:00:08.167ZLast Flush: 1011 1000 1011
128062024-02-07T19:00:08.168ZDownstairs last five completed: 1000
128072024-02-07T19:00:08.168ZUpstairs last five completed: 30 29 28 27 26
128082024-02-07T19:00:08.168Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:07.372973227Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1599}
128092024-02-07T19:00:08.168ZWrote out file "/tmp/test_repair_verify.data"
128102024-02-07T19:00:08.168Z
128112024-02-07T19:00:08.180ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128122024-02-07T19:00:08.181ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128132024-02-07T19:00:08.181Z 0 000-019 115 115 115 203 203 203 F F F
128142024-02-07T19:00:08.181Z 1 020-039 116 115 116 205 203 205 T F T <---
128152024-02-07T19:00:08.181Z 2 040-059 104 104 104 178 178 178 F F F
128162024-02-07T19:00:08.181Z 3 060-079 108 108 108 191 191 191 F F F
128172024-02-07T19:00:08.181Z 4 080-099 114 114 114 201 201 201 F F F
128182024-02-07T19:00:08.181Z 5 100-119 112 112 112 195 195 195 T F T <---
128192024-02-07T19:00:08.181Z 6 120-139 115 115 115 203 203 203 F F F
128202024-02-07T19:00:08.181Z 7 140-159 115 115 115 203 203 203 T F T <---
128212024-02-07T19:00:08.181Z 8 160-179 114 114 114 200 200 200 T F T <---
128222024-02-07T19:00:08.182Z 9 180-199 114 114 114 202 202 202 T F T <---
128232024-02-07T19:00:08.182Z 10 200-219 116 106 116 205 187 205 T F T <---
128242024-02-07T19:00:08.182Z 11 220-239 115 115 115 203 203 203 F F F
128252024-02-07T19:00:08.182Z 12 240-259 104 104 104 180 180 180 F F F
128262024-02-07T19:00:08.182Z 13 260-279 116 114 116 205 202 205 F F F <---
128272024-02-07T19:00:08.182Z 14 280-299 114 114 114 202 202 202 F F F
128282024-02-07T19:00:08.182Z 15 300-319 114 114 114 202 202 202 F F F
128292024-02-07T19:00:08.182Z 16 320-339 115 115 115 203 203 203 F F F
128302024-02-07T19:00:08.183Z 17 340-359 108 108 108 191 191 191 T F T <---
128312024-02-07T19:00:08.183Z 18 360-379 108 108 108 190 190 190 F F F
128322024-02-07T19:00:08.183Z 19 380-399 115 115 115 203 203 203 F F F
128332024-02-07T19:00:08.183Z 20 400-419 115 115 115 203 203 203 F F F
128342024-02-07T19:00:08.183Z 21 420-439 111 111 111 194 194 194 T F T <---
128352024-02-07T19:00:08.183Z 22 440-459 110 110 110 193 193 193 T F T <---
128362024-02-07T19:00:08.183Z 23 460-479 106 106 106 186 186 186 F F F
128372024-02-07T19:00:08.184Z 24 480-499 108 108 108 191 191 191 F F F
128382024-02-07T19:00:08.184Z 25 500-519 116 108 116 205 191 205 F F F <---
128392024-02-07T19:00:08.184Z 26 520-539 116 113 116 205 198 205 F F F <---
128402024-02-07T19:00:08.184Z 27 540-559 112 112 112 195 195 195 F F F
128412024-02-07T19:00:08.184Z 28 560-579 110 110 110 193 193 193 F F F
128422024-02-07T19:00:08.184Z 29 580-599 108 108 108 190 190 190 F F F
128432024-02-07T19:00:08.184ZMax gen: 116, Max flush: 205
128442024-02-07T19:00:08.184ZError: Difference in extent metadata found!
128452024-02-07T19:00:08.189ZA Difference in extent metadata is expected here
128462024-02-07T19:00:08.189ZOn loop 58
128472024-02-07T19:00:08.190Z
128482024-02-07T19:00:08.190ZVerifying data now
128492024-02-07T19:00:12.780ZNo change after verify
128502024-02-07T19:00:12.780ZLoop: 58 Downstairs dump after verify (and repair):
128512024-02-07T19:00:12.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128522024-02-07T19:00:12.792Z 0 000-019 115 115 115 203 203 203 F F F
128532024-02-07T19:00:12.792Z 1 020-039 117 117 117 206 206 206 F F F
128542024-02-07T19:00:12.792Z 2 040-059 104 104 104 178 178 178 F F F
128552024-02-07T19:00:12.792Z 3 060-079 108 108 108 191 191 191 F F F
128562024-02-07T19:00:12.792Z 4 080-099 114 114 114 201 201 201 F F F
128572024-02-07T19:00:12.792Z 5 100-119 117 117 117 206 206 206 F F F
128582024-02-07T19:00:12.792Z 6 120-139 115 115 115 203 203 203 F F F
128592024-02-07T19:00:12.792Z 7 140-159 117 117 117 206 206 206 F F F
128602024-02-07T19:00:12.792Z 8 160-179 117 117 117 206 206 206 F F F
128612024-02-07T19:00:12.808Z 9 180-199 117 117 117 206 206 206 F F F
128622024-02-07T19:00:12.808Z 10 200-219 117 117 117 206 206 206 F F F
128632024-02-07T19:00:12.808Z 11 220-239 115 115 115 203 203 203 F F F
128642024-02-07T19:00:12.808Z 12 240-259 104 104 104 180 180 180 F F F
128652024-02-07T19:00:12.808Z 13 260-279 116 116 116 205 205 205 F F F
128662024-02-07T19:00:12.808Z 14 280-299 114 114 114 202 202 202 F F F
128672024-02-07T19:00:12.808Z 15 300-319 114 114 114 202 202 202 F F F
128682024-02-07T19:00:12.808Z 16 320-339 115 115 115 203 203 203 F F F
128692024-02-07T19:00:12.808Z 17 340-359 117 117 117 206 206 206 F F F
128702024-02-07T19:00:12.809Z 18 360-379 108 108 108 190 190 190 F F F
128712024-02-07T19:00:12.809Z 19 380-399 115 115 115 203 203 203 F F F
128722024-02-07T19:00:12.809Z 20 400-419 115 115 115 203 203 203 F F F
128732024-02-07T19:00:12.809Z 21 420-439 117 117 117 206 206 206 F F F
128742024-02-07T19:00:12.809Z 22 440-459 117 117 117 206 206 206 F F F
128752024-02-07T19:00:12.809Z 23 460-479 106 106 106 186 186 186 F F F
128762024-02-07T19:00:12.809Z 24 480-499 108 108 108 191 191 191 F F F
128772024-02-07T19:00:12.809Z 25 500-519 116 116 116 205 205 205 F F F
128782024-02-07T19:00:12.809Z 26 520-539 116 116 116 205 205 205 F F F
128792024-02-07T19:00:12.809Z 27 540-559 112 112 112 195 195 195 F F F
128802024-02-07T19:00:12.809Z 28 560-579 110 110 110 193 193 193 F F F
128812024-02-07T19:00:12.809Z 29 580-599 108 108 108 190 190 190 F F F
128822024-02-07T19:00:12.810ZMax gen: 117, Max flush: 206
128832024-02-07T19:00:12.810Z
128842024-02-07T19:00:12.810ZBegin loop 59
128852024-02-07T19:00:12.810ZDownstairs to restart: 1
128862024-02-07T19:00:12.817Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035019932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607}
128872024-02-07T19:00:12.817Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035266568Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607}
128882024-02-07T19:00:12.817Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035286965Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607}
128892024-02-07T19:00:12.818Z{"msg":"Crucible c6d27d06-554c-411b-b020-0df2163a1660 has session id: 7b29ff0f-e76c-4bbe-a168-6419514a12fc","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035320221Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607Crucible runtime is spawned
128902024-02-07T19:00:12.818Z,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128912024-02-07T19:00:12.818Z{"msg":"Upstairs opts: Upstairs UUID: c6d27d06-554c-411b-b020-0df2163a1660, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035363615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128922024-02-07T19:00:12.818Z{"msg":"Crucible stats registered with UUID: c6d27d06-554c-411b-b020-0df2163a1660","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035391211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128932024-02-07T19:00:12.818Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035410689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607}
128942024-02-07T19:00:12.818Z{"msg":"c6d27d06-554c-411b-b020-0df2163a1660 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035503296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128952024-02-07T19:00:12.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035530003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128962024-02-07T19:00:12.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035546381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128972024-02-07T19:00:12.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035567268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128982024-02-07T19:00:12.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035581536Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
128992024-02-07T19:00:12.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035594854Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129002024-02-07T19:00:12.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035629439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129012024-02-07T19:00:12.819Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035647837Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129022024-02-07T19:00:12.819Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035663055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129032024-02-07T19:00:12.819Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035692391Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129042024-02-07T19:00:12.820Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:12.035835442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129052024-02-07T19:00:12.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035882995Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129062024-02-07T19:00:12.820Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:12.03591763Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129072024-02-07T19:00:12.820Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:12.035950136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129082024-02-07T19:00:12.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.035964124Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129092024-02-07T19:00:12.820Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.03599478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129102024-02-07T19:00:12.820Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.036022116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129112024-02-07T19:00:12.821Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.036091697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129122024-02-07T19:00:12.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.036678717Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129132024-02-07T19:00:12.821Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.036715303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129142024-02-07T19:00:12.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.03673253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129152024-02-07T19:00:12.822Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.036791782Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129162024-02-07T19:00:12.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.037312991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129172024-02-07T19:00:12.822Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.037460222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129182024-02-07T19:00:12.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.037849209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129192024-02-07T19:00:12.822Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.037894353Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129202024-02-07T19:00:12.822Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.03791699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129212024-02-07T19:00:12.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.038008348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129222024-02-07T19:00:12.822Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.038038164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129232024-02-07T19:00:12.822Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:12.038053551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129242024-02-07T19:00:13.817Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.036814989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129252024-02-07T19:00:13.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.036999514Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129262024-02-07T19:00:13.829Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.037800275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129272024-02-07T19:00:13.829Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.03784294Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129282024-02-07T19:00:13.829Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.038362099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129292024-02-07T19:00:13.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.038980146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129302024-02-07T19:00:13.829Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039018341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129312024-02-07T19:00:13.829Z{"msg":"[0]R flush_numbers[0..12]: [203, 206, 178, 191, 201, 206, 203, 206, 206, 206, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039035998Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129322024-02-07T19:00:13.830Z{"msg":"[0]R generation[0..12]: [115, 117, 104, 108, 114, 117, 115, 117, 117, 117, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039051876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129332024-02-07T19:00:13.830Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039074023Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129342024-02-07T19:00:13.830Z{"msg":"[1]R flush_numbers[0..12]: [203, 206, 178, 191, 201, 206, 203, 206, 206, 206, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039089651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129352024-02-07T19:00:13.830Z{"msg":"[1]R generation[0..12]: [115, 117, 104, 108, 114, 117, 115, 117, 117, 117, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039110468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129362024-02-07T19:00:13.830Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039125376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129372024-02-07T19:00:13.830Z{"msg":"[2]R flush_numbers[0..12]: [203, 206, 178, 191, 201, 206, 203, 206, 206, 206, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039146243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129382024-02-07T19:00:13.830Z{"msg":"[2]R generation[0..12]: [115, 117, 104, 108, 114, 117, 115, 117, 117, 117, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039160851Z","hostname":"Wait for a query_work_queue command to finish before sending IO
129392024-02-07T19:00:13.831Zip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129402024-02-07T19:00:13.831Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039189798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129412024-02-07T19:00:13.831Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039203656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129422024-02-07T19:00:13.831Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039224293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129432024-02-07T19:00:13.831Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039238031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129442024-02-07T19:00:13.831Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039257918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129452024-02-07T19:00:13.831Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039272036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129462024-02-07T19:00:13.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039288694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129472024-02-07T19:00:13.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.03931495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129482024-02-07T19:00:13.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039330408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129492024-02-07T19:00:13.832Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039355915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129502024-02-07T19:00:13.832Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039368993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129512024-02-07T19:00:13.832Z{"msg":"c6d27d06-554c-411b-b020-0df2163a1660 is now active with session: 7b29ff0f-e76c-4bbe-a168-6419514a12fc","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039382181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129522024-02-07T19:00:13.832Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039402429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607,"session_id":"7b29ff0f-e76c-4bbe-a168-6419514a12fc"}
129532024-02-07T19:00:13.832Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:13.039420776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607}
129542024-02-07T19:00:13.832ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129552024-02-07T19:00:13.832ZLoading write count information from file "/tmp/test_repair_verify.data"
129562024-02-07T19:00:13.833ZRun Repair workload
129572024-02-07T19:00:13.833Z01/30 Write block 405 len 4096 data: 8
129582024-02-07T19:00:13.833Z02/30 Write block 23 len 28672 data: 14 11 10 8 8 10 14
129592024-02-07T19:00:13.833Z03/30 Write block 59 len 4096 data: 9
129602024-02-07T19:00:13.834Z04/30 Read block 184 len 12288
129612024-02-07T19:00:13.834Z05/30 Flush
129622024-02-07T19:00:14.248Z06/30 Write block 177 len 24576 data: 7 7 7 9 4 4
129632024-02-07T19:00:14.248Z07/30 Read block 95 len 16384
129642024-02-07T19:00:14.248Z08/30 Flush
129652024-02-07T19:00:14.249Z09/30 Write block 161 len 24576 data: 7 9 11 13 13 10
129662024-02-07T19:00:14.249Z10/30 Read block 158 len 28672
129672024-02-07T19:00:14.249Z11/30 Read block 255 len 40960
129682024-02-07T19:00:14.249Z12/30 Flush
129692024-02-07T19:00:14.249Z13/30 Write block 430 len 8192 data: 9 9
129702024-02-07T19:00:14.249Z14/30 Read block 298 len 28672
129712024-02-07T19:00:14.249Z15/30 Flush
129722024-02-07T19:00:14.249Z16/30 Write block 369 len 36864 data: 10 12 13 9 9 8 9 9 8
129732024-02-07T19:00:14.249Z17/30 Read block 330 len 16384
129742024-02-07T19:00:14.249Z18/30 Write block 563 len 32768 data: 8 11 11 11 12 12 11 11
129752024-02-07T19:00:14.249Z19/30 Read block 36 len 12288
129762024-02-07T19:00:14.249Z20/30 Write block 22 len 28672 data: 14 15 12 11 9 9 11
129772024-02-07T19:00:14.250Z21/30 Read block 328 len 28672
129782024-02-07T19:00:14.250Z22/30 Write block 583 len 40960 data: 8 8 9 9 8 10 10 10 11 10
129792024-02-07T19:00:14.250Z23/30 Read block 224 len 8192
129802024-02-07T19:00:14.282Z24/30 Write block 369 len 24576 data: 11 13 14 10 10 9
129812024-02-07T19:00:14.282Z25/30 Write block 591 len 36864 data: 12 11 9 8 6 4 4 3 3
129822024-02-07T19:00:14.282Z26/30 Flush
129832024-02-07T19:00:14.283Z27/30 Write block 17 len 16384 data: 13 11 14 15
129842024-02-07T19:00:14.283Z28/30 Read block 437 len 32768
129852024-02-07T19:00:14.283Z29/30 Read block 99 len 28672
129862024-02-07T19:00:14.283Z30/30 Write block 164 len 36864 data: 14 14 11 10 13 13 12 12 11
129872024-02-07T19:00:14.283Z----------------------------------------------------------------
129882024-02-07T19:00:14.283Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
129892024-02-07T19:00:14.284ZStates: Active Active Active
129902024-02-07T19:00:14.284ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129912024-02-07T19:00:14.284Z 1 Acked 1000 Write 1 Done Done Done false
129922024-02-07T19:00:14.284Z 2 Acked 1001 Write 7 Done Sent Done false
129932024-02-07T19:00:14.284Z 3 Acked 1002 Write 1 Done Sent Done false
129942024-02-07T19:00:14.284Z 4 Acked 1003 Read 3 Done Sent Done false
129952024-02-07T19:00:14.284Z 5 Acked 1004 Flush 0 Done Sent Done false
129962024-02-07T19:00:14.284Z 6 Acked 1005 Write 6 Done Sent Done false
129972024-02-07T19:00:14.284Z 7 Acked 1006 Read 4 Done Sent Done false
129982024-02-07T19:00:14.284Z 8 Acked 1007 Flush 0 Done Sent Done false
129992024-02-07T19:00:14.284Z 9 Acked 1008 Write 6 Done Sent Done false
130002024-02-07T19:00:14.284Z 10 Acked 1009 Read 7 Done Sent Done false
130012024-02-07T19:00:14.284Z 11 Acked 1010 Read 10 Done Sent Done false
130022024-02-07T19:00:14.285Z 12 Acked 1011 Flush 0 Done Sent Done false
130032024-02-07T19:00:14.285Z 13 Acked 1012 Write 2 Done Sent Done false
130042024-02-07T19:00:14.285Z 14 Acked 1013 Read 7 Done Sent Done false
130052024-02-07T19:00:14.285Z 15 Acked 1014 Flush 0 Done Sent Done false
130062024-02-07T19:00:14.285Z 16 Acked 1015 Write 9 Done Sent Done false
130072024-02-07T19:00:14.285Z 17 Acked 1016 Read 4 Done Sent Done false
130082024-02-07T19:00:14.285Z 18 Acked 1017 Write 8 Done Sent Done false
130092024-02-07T19:00:14.285Z 19 Acked 1018 Read 3 Done Sent Done false
130102024-02-07T19:00:14.285Z 20 Acked 1019 Write 7 Done Sent Done false
130112024-02-07T19:00:14.286Z 21 Acked 1020 Read 7 Done Sent Done false
130122024-02-07T19:00:14.286Z 22 Acked 1021 Write 10 Done Sent Done false
130132024-02-07T19:00:14.286Z 23 Acked 1022 Read 2 Done Sent Done false
130142024-02-07T19:00:14.286Z 24 Acked 1023 Write 6 Done Sent Done false
130152024-02-07T19:00:14.286Z 25 Acked 1024 Write 9 Done Sent Done false
130162024-02-07T19:00:14.286Z 26 Acked 1025 Flush 0 Done Sent Done false
130172024-02-07T19:00:14.286Z 27 Acked 1026 Write 4 Done Sent Done false
130182024-02-07T19:00:14.287Z 28 Acked 1027 Read 8 Done Sent Done false
130192024-02-07T19:00:14.287Z 29 Acked 1028 Read 7 Done Sent Done false
130202024-02-07T19:00:14.287Z 30 Acked 1029 Write 9 Sent Sent Sent false
130212024-02-07T19:00:14.287Z STATES DS:0 DS:1 DS:2 TOTAL
130222024-02-07T19:00:14.287Z New 0 0 0 0
130232024-02-07T19:00:14.287Z Sent 1 29 1 31
130242024-02-07T19:00:14.287Z Done 29 1 29 59
130252024-02-07T19:00:14.287Z Skipped 0 0 0 0
130262024-02-07T19:00:14.287Z Error 0 0 0 0
130272024-02-07T19:00:14.287ZLast Flush: 1025 0 1025
130282024-02-07T19:00:14.288ZDownstairs last five completed:
130292024-02-07T19:00:14.288ZUpstairs last five completed: 30 29 28 27 26
130302024-02-07T19:00:14.288Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:13.072935194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1607}
130312024-02-07T19:00:14.288ZWrote out file "/tmp/test_repair_verify.data"
130322024-02-07T19:00:14.288Z
130332024-02-07T19:00:14.288ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130342024-02-07T19:00:14.288ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130352024-02-07T19:00:14.288Z 0 000-019 115 115 115 203 203 203 T F T <---
130362024-02-07T19:00:14.289Z 1 020-039 118 117 118 211 206 211 T F T <---
130372024-02-07T19:00:14.289Z 2 040-059 118 104 118 207 178 207 F F F <---
130382024-02-07T19:00:14.289Z 3 060-079 108 108 108 191 191 191 F F F
130392024-02-07T19:00:14.289Z 4 080-099 114 114 114 201 201 201 F F F
130402024-02-07T19:00:14.289Z 5 100-119 117 117 117 206 206 206 F F F
130412024-02-07T19:00:14.289Z 6 120-139 115 115 115 203 203 203 F F F
130422024-02-07T19:00:14.289Z 7 140-159 117 117 117 206 206 206 F F F
130432024-02-07T19:00:14.290Z 8 160-179 118 117 118 209 206 209 T F T <---
130442024-02-07T19:00:14.290Z 9 180-199 118 117 118 208 206 208 F F F <---
130452024-02-07T19:00:14.290Z 10 200-219 117 117 117 206 206 206 F F F
130462024-02-07T19:00:14.291Z 11 220-239 115 115 115 203 203 203 F F F
130472024-02-07T19:00:14.291Z 12 240-259 104 104 104 180 180 180 F F F
130482024-02-07T19:00:14.291Z 13 260-279 116 116 116 205 205 205 F F F
130492024-02-07T19:00:14.291Z 14 280-299 114 114 114 202 202 202 F F F
130502024-02-07T19:00:14.291Z 15 300-319 114 114 114 202 202 202 F F F
130512024-02-07T19:00:14.291Z 16 320-339 115 115 115 203 203 203 F F F
130522024-02-07T19:00:14.291Z 17 340-359 117 117 117 206 206 206 F F F
130532024-02-07T19:00:14.291Z 18 360-379 118 108 118 211 190 211 F F F <---
130542024-02-07T19:00:14.291Z 19 380-399 115 115 115 203 203 203 F F F
130552024-02-07T19:00:14.291Z 20 400-419 118 115 118 207 203 207 F T F <---
130562024-02-07T19:00:14.291Z 21 420-439 118 117 118 210 206 210 F F F <---
130572024-02-07T19:00:14.291Z 22 440-459 117 117 117 206 206 206 F F F
130582024-02-07T19:00:14.291Z 23 460-479 106 106 106 186 186 186 F F F
130592024-02-07T19:00:14.292Z 24 480-499 108 108 108 191 191 191 F F F
130602024-02-07T19:00:14.292Z 25 500-519 116 116 116 205 205 205 F F F
130612024-02-07T19:00:14.292Z 26 520-539 116 116 116 205 205 205 F F F
130622024-02-07T19:00:14.292Z 27 540-559 112 112 112 195 195 195 F F F
130632024-02-07T19:00:14.292Z 28 560-579 118 110 118 211 193 211 F F F <---
130642024-02-07T19:00:14.292Z 29 580-599 118 108 118 211 190 211 F F F <---
130652024-02-07T19:00:14.292ZMax gen: 118, Max flush: 211
130662024-02-07T19:00:14.292ZError: Difference in extent metadata found!
130672024-02-07T19:00:14.292ZA Difference in extent metadata is expected here
130682024-02-07T19:00:14.292ZOn loop 59
130692024-02-07T19:00:14.292Z
130702024-02-07T19:00:14.292ZVerifying data now
130712024-02-07T19:00:19.379ZNo change after verify
130722024-02-07T19:00:19.379ZLoop: 59 Downstairs dump after verify (and repair):
130732024-02-07T19:00:19.395ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130742024-02-07T19:00:19.395Z 0 000-019 119 119 119 212 212 212 F F F
130752024-02-07T19:00:19.395Z 1 020-039 119 119 119 212 212 212 F F F
130762024-02-07T19:00:19.395Z 2 040-059 118 118 118 207 207 207 F F F
130772024-02-07T19:00:19.395Z 3 060-079 108 108 108 191 191 191 F F F
130782024-02-07T19:00:19.395Z 4 080-099 114 114 114 201 201 201 F F F
130792024-02-07T19:00:19.396Z 5 100-119 117 117 117 206 206 206 F F F
130802024-02-07T19:00:19.396Z 6 120-139 115 115 115 203 203 203 F F F
130812024-02-07T19:00:19.396Z 7 140-159 117 117 117 206 206 206 F F F
130822024-02-07T19:00:19.396Z 8 160-179 119 119 119 212 212 212 F F F
130832024-02-07T19:00:19.396Z 9 180-199 118 118 118 208 208 208 F F F
130842024-02-07T19:00:19.396Z 10 200-219 117 117 117 206 206 206 F F F
130852024-02-07T19:00:19.396Z 11 220-239 115 115 115 203 203 203 F F F
130862024-02-07T19:00:19.396Z 12 240-259 104 104 104 180 180 180 F F F
130872024-02-07T19:00:19.396Z 13 260-279 116 116 116 205 205 205 F F F
130882024-02-07T19:00:19.396Z 14 280-299 114 114 114 202 202 202 F F F
130892024-02-07T19:00:19.397Z 15 300-319 114 114 114 202 202 202 F F F
130902024-02-07T19:00:19.397Z 16 320-339 115 115 115 203 203 203 F F F
130912024-02-07T19:00:19.397Z 17 340-359 117 117 117 206 206 206 F F F
130922024-02-07T19:00:19.397Z 18 360-379 118 118 118 211 211 211 F F F
130932024-02-07T19:00:19.397Z 19 380-399 115 115 115 203 203 203 F F F
130942024-02-07T19:00:19.397Z 20 400-419 118 118 118 207 207 207 F F F
130952024-02-07T19:00:19.397Z 21 420-439 118 118 118 210 210 210 F F F
130962024-02-07T19:00:19.401Z 22 440-459 117 117 117 206 206 206 F F F
130972024-02-07T19:00:19.401Z 23 460-479 106 106 106 186 186 186 F F F
130982024-02-07T19:00:19.401Z 24 480-499 108 108 108 191 191 191 F F F
130992024-02-07T19:00:19.402Z 25 500-519 116 116 116 205 205 205 F F F
131002024-02-07T19:00:19.402Z 26 520-539 116 116 116 205 205 205 F F F
131012024-02-07T19:00:19.402Z 27 540-559 112 112 112 195 195 195 F F F
131022024-02-07T19:00:19.402Z 28 560-579 118 118 118 211 211 211 F F F
131032024-02-07T19:00:19.402Z 29 580-599 118 118 118 211 211 211 F F F
131042024-02-07T19:00:19.402ZMax gen: 119, Max flush: 212
131052024-02-07T19:00:19.402Z
131062024-02-07T19:00:19.402ZBegin loop 60
131072024-02-07T19:00:19.402ZDownstairs to restart: 0
131082024-02-07T19:00:19.419Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641269073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615}
131092024-02-07T19:00:19.426Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641557774Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615}
131102024-02-07T19:00:19.426Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
131112024-02-07T19:00:19.426Z,"time":"2024-02-07T19:00:18.641634633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615}
131122024-02-07T19:00:19.426Z{"msg":"Crucible 2147b601-f1ef-4d8e-8372-54773648f837 has session id: f226c983-eedc-45c3-8843-b622ff726803","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641687346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131132024-02-07T19:00:19.426Z{"msg":"Upstairs opts: Upstairs UUID: 2147b601-f1ef-4d8e-8372-54773648f837, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641748108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131142024-02-07T19:00:19.426Z{"msg":"Crucible stats registered with UUID: 2147b601-f1ef-4d8e-8372-54773648f837","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641791642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131152024-02-07T19:00:19.426Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641831357Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615}
131162024-02-07T19:00:19.426Z{"msg":"2147b601-f1ef-4d8e-8372-54773648f837 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641857843Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131172024-02-07T19:00:19.426Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.641968208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131182024-02-07T19:00:19.427Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.64202368Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131192024-02-07T19:00:19.427Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642118158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131202024-02-07T19:00:19.427Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642159532Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131212024-02-07T19:00:19.427Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642196507Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131222024-02-07T19:00:19.427Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642236992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131232024-02-07T19:00:19.427Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642264348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131242024-02-07T19:00:19.427Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642303883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131252024-02-07T19:00:19.427Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642333219Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131262024-02-07T19:00:19.428Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642371793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131272024-02-07T19:00:19.428Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642419737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131282024-02-07T19:00:19.428Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:18.642462661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131292024-02-07T19:00:19.428Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:18.642495367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131302024-02-07T19:00:19.428Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:18.642539961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131312024-02-07T19:00:19.428Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642566027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131322024-02-07T19:00:19.428Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642606902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131332024-02-07T19:00:19.429Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.642715697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131342024-02-07T19:00:19.429Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.643225628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131352024-02-07T19:00:19.429Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.643262523Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131362024-02-07T19:00:19.429Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.643278981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131372024-02-07T19:00:19.429Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.643293139Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131382024-02-07T19:00:19.429Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.64379918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131392024-02-07T19:00:19.429Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.643914535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131402024-02-07T19:00:19.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.644360735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131412024-02-07T19:00:19.430Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.644398689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131422024-02-07T19:00:19.430Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.644428615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131432024-02-07T19:00:19.430Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.644456431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131442024-02-07T19:00:19.430Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.64447192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131452024-02-07T19:00:19.430Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:18.644493637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131462024-02-07T19:00:20.423Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.644352395Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131472024-02-07T19:00:20.423Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.644583504Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131482024-02-07T19:00:20.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.645537115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131492024-02-07T19:00:20.430Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.645576969Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131502024-02-07T19:00:20.430Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646132585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131512024-02-07T19:00:20.430Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646750181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131522024-02-07T19:00:20.430Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646794365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131532024-02-07T19:00:20.430Z{"msg":"[0]R flush_numbers[0..12]: [212, 212, 207, 191, 201, 206, 203, 206, 212, 208, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646819612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131542024-02-07T19:00:20.430Z{"msg":"[0]R generation[0..12]: [119, 119, 118, 108, 114, 117, 115, 117, 119, 118, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646858866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131552024-02-07T19:00:20.430Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646901601Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131562024-02-07T19:00:20.430Z{"msg":"[1]R flush_numbers[0..12]: [212, 212, 207, 191, 201, 206, 203, 206, 212, 208, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646934766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131572024-02-07T19:00:20.431ZWait for a query_work_queue command to finish before sending IO
131582024-02-07T19:00:20.431Z{"msg":"[1]R generation[0..12]: [119, 119, 118, 108, 114, 117, 115, 117, 119, 118, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.646985649Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131592024-02-07T19:00:20.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647016765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131602024-02-07T19:00:20.431Z{"msg":"[2]R flush_numbers[0..12]: [212, 212, 207, 191, 201, 206, 203, 206, 212, 208, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.64704959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131612024-02-07T19:00:20.431Z{"msg":"[2]R generation[0..12]: [119, 119, 118, 108, 114, 117, 115, 117, 119, 118, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647066568Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131622024-02-07T19:00:20.431Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647102793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131632024-02-07T19:00:20.431Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647139038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131642024-02-07T19:00:20.431Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647157236Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131652024-02-07T19:00:20.431Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647171544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131662024-02-07T19:00:20.431Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.64720372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131672024-02-07T19:00:20.432Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647239575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131682024-02-07T19:00:20.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647264401Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131692024-02-07T19:00:20.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647279699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131702024-02-07T19:00:20.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647313025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131712024-02-07T19:00:20.432Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.64734776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131722024-02-07T19:00:20.432Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647362738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131732024-02-07T19:00:20.432Z{"msg":"2147b601-f1ef-4d8e-8372-54773648f837 is now active with session: f226c983-eedc-45c3-8843-b622ff726803","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.647389284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131742024-02-07T19:00:20.432Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:19.64742241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615,"session_id":"f226c983-eedc-45c3-8843-b622ff726803"}
131752024-02-07T19:00:20.432Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131762024-02-07T19:00:20.433Z,"time":"2024-02-07T19:00:19.647447277Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615}
131772024-02-07T19:00:20.433ZLoading write count information from file "/tmp/test_repair_verify.data"
131782024-02-07T19:00:20.433ZRun Repair workload
131792024-02-07T19:00:20.433Z01/30 Write block 350 len 24576 data: 12 11 7 6 10 7
131802024-02-07T19:00:20.433Z02/30 Write block 267 len 4096 data: 13
131812024-02-07T19:00:20.437Z03/30 Write block 508 len 24576 data: 10 10 12 9 12 9
131822024-02-07T19:00:20.437Z04/30 Write block 0 len 8192 data: 2 3
131832024-02-07T19:00:20.438Z05/30 Write block 8 len 16384 data: 14 15 11 11
131842024-02-07T19:00:20.438Z06/30 Read block 456 len 24576
131852024-02-07T19:00:20.438Z07/30 Write block 370 len 24576 data: 14 15 11 11 10 10
131862024-02-07T19:00:20.438Z08/30 Write block 552 len 4096 data: 8
131872024-02-07T19:00:20.438Z09/30 Read block 337 len 40960
131882024-02-07T19:00:20.438Z10/30 Read block 273 len 4096
131892024-02-07T19:00:20.438Z11/30 Read block 197 len 20480
131902024-02-07T19:00:20.438Z12/30 Read block 489 len 28672
131912024-02-07T19:00:20.438Z13/30 Write block 376 len 4096 data: 10
131922024-02-07T19:00:20.439Z14/30 Write block 352 len 28672 data: 8 7 11 8 5 3 5
131932024-02-07T19:00:20.439Z15/30 Read block 349 len 8192
131942024-02-07T19:00:20.456Z16/30 Write block 376 len 24576 data: 11 9 9 7 8 7
131952024-02-07T19:00:20.456Z17/30 Write block 512 len 36864 data: 13 10 11 10 9 6 8 8 8
131962024-02-07T19:00:20.456Z18/30 Read block 430 len 12288
131972024-02-07T19:00:20.456Z19/30 Write block 410 len 8192 data: 13 14
131982024-02-07T19:00:20.456Z20/30 Write block 253 len 8192 data: 7 7
131992024-02-07T19:00:20.456Z21/30 Read block 23 len 32768
132002024-02-07T19:00:20.457Z22/30 Write block 304 len 24576 data: 6 6 5 4 7 8
132012024-02-07T19:00:20.457Z23/30 Read block 340 len 24576
132022024-02-07T19:00:20.457Z24/30 Write block 284 len 8192 data: 13 13
132032024-02-07T19:00:20.457Z25/30 Write block 249 len 12288 data: 8 7 6
132042024-02-07T19:00:20.457Z26/30 Read block 283 len 24576
132052024-02-07T19:00:20.457Z27/30 Read block 25 len 12288
132062024-02-07T19:00:20.457Z28/30 Read block 117 len 24576
132072024-02-07T19:00:20.457Z29/30 Write block 305 len 32768 data: 7 6 5 8 9 9 9 7
132082024-02-07T19:00:20.458Z30/30 Read block 448 len 8192
132092024-02-07T19:00:20.458Z----------------------------------------------------------------
132102024-02-07T19:00:20.458Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
132112024-02-07T19:00:20.458ZStates: Active Active Active
132122024-02-07T19:00:20.458ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132132024-02-07T19:00:20.459Z 1 Acked 1000 Write 6 Sent Done Done false
132142024-02-07T19:00:20.459Z 2 Acked 1001 Write 1 Sent Done Done false
132152024-02-07T19:00:20.459Z 3 Acked 1002 Write 6 Sent Done Done false
132162024-02-07T19:00:20.459Z 4 Acked 1003 Write 2 Sent Done Done false
132172024-02-07T19:00:20.459Z 5 Acked 1004 Write 4 Sent Done Done false
132182024-02-07T19:00:20.459Z 6 Acked 1005 Read 6 Sent Done Done false
132192024-02-07T19:00:20.459Z 7 Acked 1006 Write 6 Sent Done Done false
132202024-02-07T19:00:20.460Z 8 Acked 1007 Write 1 Sent Done Done false
132212024-02-07T19:00:20.460Z 9 Acked 1008 Read 10 Sent Done Done false
132222024-02-07T19:00:20.460Z 10 Acked 1009 Read 1 Sent Done Done false
132232024-02-07T19:00:20.460Z 11 Acked 1010 Read 5 Sent Done Done false
132242024-02-07T19:00:20.460Z 12 Acked 1011 Read 7 Sent Done Done false
132252024-02-07T19:00:20.460Z 13 Acked 1012 Write 1 Sent Done Done false
132262024-02-07T19:00:20.460Z 14 Acked 1013 Write 7 Sent Done Done false
132272024-02-07T19:00:20.461Z 15 Acked 1014 Read 2 Sent Done Done false
132282024-02-07T19:00:20.461Z 16 Acked 1015 Write 6 Sent Done Done false
132292024-02-07T19:00:20.461Z 17 Acked 1016 Write 9 Sent Done Done false
132302024-02-07T19:00:20.461Z 18 Acked 1017 Read 3 Sent Done Done false
132312024-02-07T19:00:20.461Z 19 Acked 1018 Write 2 Sent Done Done false
132322024-02-07T19:00:20.462Z 20 Acked 1019 Write 2 Sent Done Done false
132332024-02-07T19:00:20.462Z 21 Acked 1020 Read 8 Sent Done Done false
132342024-02-07T19:00:20.462Z 22 Acked 1021 Write 6 Sent Done Done false
132352024-02-07T19:00:20.462Z 23 Acked 1022 Read 6 Sent Done Done false
132362024-02-07T19:00:20.462Z 24 Acked 1023 Write 2 Sent Done Done false
132372024-02-07T19:00:20.462Z 25 Acked 1024 Write 3 Sent Done Done false
132382024-02-07T19:00:20.462Z 26 Acked 1025 Read 6 Sent Done Done false
132392024-02-07T19:00:20.462Z 27 Acked 1026 Read 3 Sent Done Done false
132402024-02-07T19:00:20.463Z 28 Acked 1027 Read 6 Sent Done Done false
132412024-02-07T19:00:20.463Z 29 Acked 1028 Write 8 Sent Done Done false
132422024-02-07T19:00:20.463Z 30 Acked 1029 Read 2 Sent Done Sent false
132432024-02-07T19:00:20.463Z STATES DS:0 DS:1 DS:2 TOTAL
132442024-02-07T19:00:20.463Z New 0 0 0 0
132452024-02-07T19:00:20.463Z Sent 30 0 1 31
132462024-02-07T19:00:20.464Z Done 0 30 29 59
132472024-02-07T19:00:20.464Z Skipped 0 0 0 0
132482024-02-07T19:00:20.464Z Error 0 0 0 0
132492024-02-07T19:00:20.464ZLast Flush: 0 0 0
132502024-02-07T19:00:20.464ZDownstairs last five completed:
132512024-02-07T19:00:20.464ZUpstairs last five completed: 30 29 28 27 26
132522024-02-07T19:00:20.464Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:19.673311418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1615}
132532024-02-07T19:00:20.464ZWrote out file "/tmp/test_repair_verify.data"
132542024-02-07T19:00:20.475Z
132552024-02-07T19:00:20.475ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132562024-02-07T19:00:20.483ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132572024-02-07T19:00:20.484Z 0 000-019 119 119 119 212 212 212 F T T <---
132582024-02-07T19:00:20.484Z 1 020-039 119 119 119 212 212 212 F F F
132592024-02-07T19:00:20.484Z 2 040-059 118 118 118 207 207 207 F F F
132602024-02-07T19:00:20.485Z 3 060-079 108 108 108 191 191 191 F F F
132612024-02-07T19:00:20.485Z 4 080-099 114 114 114 201 201 201 F F F
132622024-02-07T19:00:20.485Z 5 100-119 117 117 117 206 206 206 F F F
132632024-02-07T19:00:20.485Z 6 120-139 115 115 115 203 203 203 F F F
132642024-02-07T19:00:20.485Z 7 140-159 117 117 117 206 206 206 F F F
132652024-02-07T19:00:20.485Z 8 160-179 119 119 119 212 212 212 F F F
132662024-02-07T19:00:20.485Z 9 180-199 118 118 118 208 208 208 F F F
132672024-02-07T19:00:20.485Z 10 200-219 117 117 117 206 206 206 F F F
132682024-02-07T19:00:20.486Z 11 220-239 115 115 115 203 203 203 F F F
132692024-02-07T19:00:20.486Z 12 240-259 104 104 104 180 180 180 F T T <---
132702024-02-07T19:00:20.486Z 13 260-279 116 116 116 205 205 205 F T T <---
132712024-02-07T19:00:20.486Z 14 280-299 114 114 114 202 202 202 F T T <---
132722024-02-07T19:00:20.486Z 15 300-319 114 114 114 202 202 202 F T T <---
132732024-02-07T19:00:20.486Z 16 320-339 115 115 115 203 203 203 F F F
132742024-02-07T19:00:20.486Z 17 340-359 117 117 117 206 206 206 F T T <---
132752024-02-07T19:00:20.486Z 18 360-379 118 118 118 211 211 211 F T T <---
132762024-02-07T19:00:20.486Z 19 380-399 115 115 115 203 203 203 F T T <---
132772024-02-07T19:00:20.486Z 20 400-419 118 118 118 207 207 207 F T T <---
132782024-02-07T19:00:20.486Z 21 420-439 118 118 118 210 210 210 F F F
132792024-02-07T19:00:20.486Z 22 440-459 117 117 117 206 206 206 F F F
132802024-02-07T19:00:20.486Z 23 460-479 106 106 106 186 186 186 F F F
132812024-02-07T19:00:20.487Z 24 480-499 108 108 108 191 191 191 F F F
132822024-02-07T19:00:20.487Z 25 500-519 116 116 116 205 205 205 F T T <---
132832024-02-07T19:00:20.487Z 26 520-539 116 116 116 205 205 205 F T T <---
132842024-02-07T19:00:20.487Z 27 540-559 112 112 112 195 195 195 F T T <---
132852024-02-07T19:00:20.487Z 28 560-579 118 118 118 211 211 211 F F F
132862024-02-07T19:00:20.487Z 29 580-599 118 118 118 211 211 211 F F F
132872024-02-07T19:00:20.487ZMax gen: 119, Max flush: 212
132882024-02-07T19:00:20.487ZError: Difference in extent metadata found!
132892024-02-07T19:00:20.487ZA Difference in extent metadata is expected here
132902024-02-07T19:00:20.487ZOn loop 60
132912024-02-07T19:00:20.487Z
132922024-02-07T19:00:20.492ZVerifying data now
132932024-02-07T19:00:25.091ZNo change after verify
132942024-02-07T19:00:25.091ZLoop: 60 Downstairs dump after verify (and repair):
132952024-02-07T19:00:25.096ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132962024-02-07T19:00:25.097Z 0 000-019 120 120 120 213 213 213 F F F
132972024-02-07T19:00:25.097Z 1 020-039 119 119 119 212 212 212 F F F
132982024-02-07T19:00:25.097Z 2 040-059 118 118 118 207 207 207 F F F
132992024-02-07T19:00:25.097Z 3 060-079 108 108 108 191 191 191 F F F
133002024-02-07T19:00:25.097Z 4 080-099 114 114 114 201 201 201 F F F
133012024-02-07T19:00:25.097Z 5 100-119 117 117 117 206 206 206 F F F
133022024-02-07T19:00:25.097Z 6 120-139 115 115 115 203 203 203 F F F
133032024-02-07T19:00:25.098Z 7 140-159 117 117 117 206 206 206 F F F
133042024-02-07T19:00:25.098Z 8 160-179 119 119 119 212 212 212 F F F
133052024-02-07T19:00:25.098Z 9 180-199 118 118 118 208 208 208 F F F
133062024-02-07T19:00:25.098Z 10 200-219 117 117 117 206 206 206 F F F
133072024-02-07T19:00:25.098Z 11 220-239 115 115 115 203 203 203 F F F
133082024-02-07T19:00:25.098Z 12 240-259 120 120 120 213 213 213 F F F
133092024-02-07T19:00:25.098Z 13 260-279 120 120 120 213 213 213 F F F
133102024-02-07T19:00:25.098Z 14 280-299 120 120 120 213 213 213 F F F
133112024-02-07T19:00:25.098Z 15 300-319 120 120 120 213 213 213 F F F
133122024-02-07T19:00:25.098Z 16 320-339 115 115 115 203 203 203 F F F
133132024-02-07T19:00:25.098Z 17 340-359 120 120 120 213 213 213 F F F
133142024-02-07T19:00:25.098Z 18 360-379 120 120 120 213 213 213 F F F
133152024-02-07T19:00:25.098Z 19 380-399 120 120 120 213 213 213 F F F
133162024-02-07T19:00:25.098Z 20 400-419 120 120 120 213 213 213 F F F
133172024-02-07T19:00:25.098Z 21 420-439 118 118 118 210 210 210 F F F
133182024-02-07T19:00:25.098Z 22 440-459 117 117 117 206 206 206 F F F
133192024-02-07T19:00:25.098Z 23 460-479 106 106 106 186 186 186 F F F
133202024-02-07T19:00:25.098Z 24 480-499 108 108 108 191 191 191 F F F
133212024-02-07T19:00:25.099Z 25 500-519 120 120 120 213 213 213 F F F
133222024-02-07T19:00:25.099Z 26 520-539 120 120 120 213 213 213 F F F
133232024-02-07T19:00:25.099Z 27 540-559 120 120 120 213 213 213 F F F
133242024-02-07T19:00:25.099Z 28 560-579 118 118 118 211 211 211 F F F
133252024-02-07T19:00:25.099Z 29 580-599 118 118 118 211 211 211 F F F
133262024-02-07T19:00:25.099ZMax gen: 120, Max flush: 213
133272024-02-07T19:00:25.109Z
133282024-02-07T19:00:25.109ZBegin loop 61
133292024-02-07T19:00:25.110ZDownstairs to restart: 2
133302024-02-07T19:00:25.118Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339324658Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623}
133312024-02-07T19:00:25.118ZCrucible runtime is spawned
133322024-02-07T19:00:25.125Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339673381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623}
133332024-02-07T19:00:25.125Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339721935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623}
133342024-02-07T19:00:25.126Z{"msg":"Crucible 9265bfa1-2136-42a5-b50e-0887247dd33d has session id: 8c4de0d5-8f04-42f0-a5bf-2deae66d9178","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339749151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133352024-02-07T19:00:25.126Z{"msg":"Upstairs opts: Upstairs UUID: 9265bfa1-2136-42a5-b50e-0887247dd33d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339778977Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133362024-02-07T19:00:25.126Z{"msg":"Crucible stats registered with UUID: 9265bfa1-2136-42a5-b50e-0887247dd33d","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339797585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133372024-02-07T19:00:25.126Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339825041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623}
133382024-02-07T19:00:25.126Z{"msg":"9265bfa1-2136-42a5-b50e-0887247dd33d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339840089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133392024-02-07T19:00:25.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339863985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133402024-02-07T19:00:25.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.33990526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133412024-02-07T19:00:25.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339949524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133422024-02-07T19:00:25.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.339997047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133432024-02-07T19:00:25.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340072317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133442024-02-07T19:00:25.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340110752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133452024-02-07T19:00:25.127Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340140388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133462024-02-07T19:00:25.127Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340188052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133472024-02-07T19:00:25.127Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340220947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133482024-02-07T19:00:25.127Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340282949Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133492024-02-07T19:00:25.127Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340314315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133502024-02-07T19:00:25.127Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:24.340341241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133512024-02-07T19:00:25.127Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:24.340386775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133522024-02-07T19:00:25.128Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:24.340536014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133532024-02-07T19:00:25.128Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340563271Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133542024-02-07T19:00:25.128Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340578659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133552024-02-07T19:00:25.128Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340731318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133562024-02-07T19:00:25.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340925062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133572024-02-07T19:00:25.128Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340958148Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133582024-02-07T19:00:25.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.340973865Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133592024-02-07T19:00:25.129Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.34101214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133602024-02-07T19:00:25.129Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.34167422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133612024-02-07T19:00:25.129Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.341714825Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133622024-02-07T19:00:25.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.342276049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133632024-02-07T19:00:25.129Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.342311125Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133642024-02-07T19:00:25.129Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.342326372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133652024-02-07T19:00:25.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.34233971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133662024-02-07T19:00:25.130Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.342353469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133672024-02-07T19:00:25.130Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:24.342374106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133682024-02-07T19:00:26.122Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.341867974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133692024-02-07T19:00:26.123Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.342004455Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133702024-02-07T19:00:26.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.342798398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133712024-02-07T19:00:26.131Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.342837462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133722024-02-07T19:00:26.132Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.343377639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133732024-02-07T19:00:26.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.343945722Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133742024-02-07T19:00:26.132Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.343979488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133752024-02-07T19:00:26.132Z{"msg":"[0]R flush_numbers[0..12]: [213, 212, 207, 191, 201, 206, 203, 206, 212, 208, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.34403381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133762024-02-07T19:00:26.132Z{"msg":"[0]R generation[0..12]: [120, 119, 118, 108, 114, 117, 115, 117, 119, 118, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344056078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133772024-02-07T19:00:26.132Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344082824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133782024-02-07T19:00:26.132Z{"msg":"[1]R flush_numbers[0..12]: [213, 212, 207, 191, 201, 206, 203, 206, 212, 208, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344102331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133792024-02-07T19:00:26.132Z{"msg":"[1]R generation[0..12]: [120, 119, 118, 108, 114, 117, 115, 117, 119, 118, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344127788Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133802024-02-07T19:00:26.133Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344146145Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133812024-02-07T19:00:26.133ZWait for a query_work_queue command to finish before sending IO
133822024-02-07T19:00:26.133Z{"msg":"[2]R flush_numbers[0..12]: [213, 212, 207, 191, 201, 206, 203, 206, 212, 208, 206, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344175771Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133832024-02-07T19:00:26.133Z{"msg":"[2]R generation[0..12]: [120, 119, 118, 108, 114, 117, 115, 117, 119, 118, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344195988Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133842024-02-07T19:00:26.133Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344221985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133852024-02-07T19:00:26.133Z{"msg":"Max found gen is 121","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344253251Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133862024-02-07T19:00:26.133Z{"msg":"Generation requested: 122 >= found:121","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344272968Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133872024-02-07T19:00:26.133Z{"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344316242Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133882024-02-07T19:00:26.134Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344345348Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133892024-02-07T19:00:26.134Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344363336Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133902024-02-07T19:00:26.134Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344379384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133912024-02-07T19:00:26.134Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.34440552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133922024-02-07T19:00:26.134Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344423268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133932024-02-07T19:00:26.134Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344448014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133942024-02-07T19:00:26.135Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344464062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133952024-02-07T19:00:26.135Z{"msg":"9265bfa1-2136-42a5-b50e-0887247dd33d is now active with session: 8c4de0d5-8f04-42f0-a5bf-2deae66d9178","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344488379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133962024-02-07T19:00:26.135Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344503897Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623,"session_id":"8c4de0d5-8f04-42f0-a5bf-2deae66d9178"}
133972024-02-07T19:00:26.135Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:25.344527364Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623}
133982024-02-07T19:00:26.135ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133992024-02-07T19:00:26.135ZLoading write count information from file "/tmp/test_repair_verify.data"
134002024-02-07T19:00:26.135ZRun Repair workload
134012024-02-07T19:00:26.135Z01/30 Write block 360 len 16384 data: 9 10 11 10
134022024-02-07T19:00:26.136Z02/30 Write block 445 len 28672 data: 13 11 11 11 9 9 10
134032024-02-07T19:00:26.136Z03/30 Read block 531 len 24576
134042024-02-07T19:00:26.136Z04/30 Read block 305 len 40960
134052024-02-07T19:00:26.136Z05/30 Write block 104 len 20480 data: 12 14 13 13 12
134062024-02-07T19:00:26.141Z06/30 Read block 155 len 8192
134072024-02-07T19:00:26.141Z07/30 Write block 337 len 8192 data: 8 10
134082024-02-07T19:00:26.141Z08/30 Flush
134092024-02-07T19:00:26.141Z09/30 Write block 260 len 24576 data: 9 8 10 13 14 12
134102024-02-07T19:00:26.141Z10/30 Write block 281 len 8192 data: 7 8
134112024-02-07T19:00:26.141Z11/30 Write block 235 len 32768 data: 18 17 18 17 17 18 19 21
134122024-02-07T19:00:26.141Z12/30 Flush
134132024-02-07T19:00:26.141Z13/30 Write block 512 len 28672 data: 14 11 12 11 10 7 9
134142024-02-07T19:00:26.141Z14/30 Read block 229 len 28672
134152024-02-07T19:00:26.141Z15/30 Read block 374 len 24576
134162024-02-07T19:00:26.142Z16/30 Read block 100 len 32768
134172024-02-07T19:00:26.142Z17/30 Read block 482 len 36864
134182024-02-07T19:00:26.145Z18/30 Read block 470 len 12288
134192024-02-07T19:00:26.146Z19/30 Write block 302 len 4096 data: 8
134202024-02-07T19:00:26.146Z20/30 Write block 423 len 12288 data: 6 7 12
134212024-02-07T19:00:26.146Z21/30 Write block 112 len 24576 data: 9 8 6 7 9 12
134222024-02-07T19:00:26.146Z22/30 Read block 84 len 28672
134232024-02-07T19:00:26.146Z23/30 Write block 581 len 32768 data: 9 10 9 9 10 10 9 11
134242024-02-07T19:00:26.153Z24/30 Read block 184 len 28672
134252024-02-07T19:00:26.153Z25/30 Read block 4 len 16384
134262024-02-07T19:00:26.153Z26/30 Read block 394 len 24576
134272024-02-07T19:00:26.153Z27/30 Read block 87 len 16384
134282024-02-07T19:00:26.153Z28/30 Write block 494 len 40960 data: 10 6 9 7 8 7 8 6 9 11
134292024-02-07T19:00:26.153Z29/30 Write block 139 len 12288 data: 6 6 10
134302024-02-07T19:00:26.153Z30/30 Read block 284 len 16384
134312024-02-07T19:00:26.154Z----------------------------------------------------------------
134322024-02-07T19:00:26.154Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:22
134332024-02-07T19:00:26.154ZStates: Active Active Active
134342024-02-07T19:00:26.154ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134352024-02-07T19:00:26.154Z 9 Acked 1008 Write 6 Done Done Done false
134362024-02-07T19:00:26.154Z 10 Acked 1009 Write 2 Done Done Sent false
134372024-02-07T19:00:26.154Z 11 Acked 1010 Write 8 Done Done Sent false
134382024-02-07T19:00:26.154Z 12 Acked 1011 Flush 0 Done Done Sent false
134392024-02-07T19:00:26.154Z 13 Acked 1012 Write 7 Done Done Sent false
134402024-02-07T19:00:26.154Z 14 Acked 1013 Read 7 Done Done Sent false
134412024-02-07T19:00:26.154Z 15 Acked 1014 Read 6 Done Done Sent false
134422024-02-07T19:00:26.155Z 16 Acked 1015 Read 8 Done Done Sent false
134432024-02-07T19:00:26.155Z 17 Acked 1016 Read 9 Done Done Sent false
134442024-02-07T19:00:26.155Z 18 Acked 1017 Read 3 Done Done Sent false
134452024-02-07T19:00:26.155Z 19 Acked 1018 Write 1 Done Done Sent false
134462024-02-07T19:00:26.155Z 20 Acked 1019 Write 3 Done Done Sent false
134472024-02-07T19:00:26.155Z 21 Acked 1020 Write 6 Done Done Sent false
134482024-02-07T19:00:26.155Z 22 Acked 1021 Read 7 Done Done Sent false
134492024-02-07T19:00:26.155Z 23 Acked 1022 Write 8 Done Done Sent false
134502024-02-07T19:00:26.155Z 24 Acked 1023 Read 7 Done Done Sent false
134512024-02-07T19:00:26.155Z 25 Acked 1024 Read 4 Done Done Sent false
134522024-02-07T19:00:26.155Z 26 Acked 1025 Read 6 Done Done Sent false
134532024-02-07T19:00:26.155Z 27 Acked 1026 Read 4 Done Done Sent false
134542024-02-07T19:00:26.156Z 28 Acked 1027 Write 10 Done Done Sent false
134552024-02-07T19:00:26.156Z 29 Acked 1028 Write 3 Done Sent Sent false
134562024-02-07T19:00:26.156Z 30 Acked 1029 Read 4 Done Sent Sent false
134572024-02-07T19:00:26.156Z STATES DS:0 DS:1 DS:2 TOTAL
134582024-02-07T19:00:26.156Z New 0 0 0 0
134592024-02-07T19:00:26.156Z Sent 0 2 21 23
134602024-02-07T19:00:26.156Z Done 22 20 1 43
134612024-02-07T19:00:26.156Z Skipped 0 0 0 0
134622024-02-07T19:00:26.156Z Error 0 0 0 0
134632024-02-07T19:00:26.156ZLast Flush: 1011 1011 1007
134642024-02-07T19:00:26.156ZDownstairs last five completed: 1007 1006 1005 1004 1003
134652024-02-07T19:00:26.156ZUpstairs last five completed: 30 29 28 27 26
134662024-02-07T19:00:26.156Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:25.374085116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1623}
134672024-02-07T19:00:26.157ZWrote out file "/tmp/test_repair_verify.data"
134682024-02-07T19:00:26.183Z
134692024-02-07T19:00:26.183ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134702024-02-07T19:00:26.183ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134712024-02-07T19:00:26.183Z 0 000-019 120 120 120 213 213 213 F F F
134722024-02-07T19:00:26.183Z 1 020-039 119 119 119 212 212 212 F F F
134732024-02-07T19:00:26.183Z 2 040-059 118 118 118 207 207 207 F F F
134742024-02-07T19:00:26.184Z 3 060-079 108 108 108 191 191 191 F F F
134752024-02-07T19:00:26.184Z 4 080-099 114 114 114 201 201 201 F F F
134762024-02-07T19:00:26.184Z 5 100-119 122 122 122 214 214 214 T T F <---
134772024-02-07T19:00:26.184Z 6 120-139 115 115 115 203 203 203 T T F <---
134782024-02-07T19:00:26.184Z 7 140-159 117 117 117 206 206 206 T T F <---
134792024-02-07T19:00:26.184Z 8 160-179 119 119 119 212 212 212 F F F
134802024-02-07T19:00:26.184Z 9 180-199 118 118 118 208 208 208 F F F
134812024-02-07T19:00:26.184Z 10 200-219 117 117 117 206 206 206 F F F
134822024-02-07T19:00:26.184Z 11 220-239 122 122 115 215 215 203 F F F <---
134832024-02-07T19:00:26.184Z 12 240-259 122 122 120 215 215 213 F F F <---
134842024-02-07T19:00:26.184Z 13 260-279 122 122 120 215 215 213 F F T <---
134852024-02-07T19:00:26.185Z 14 280-299 122 122 120 215 215 213 F F F <---
134862024-02-07T19:00:26.185Z 15 300-319 120 120 120 213 213 213 T T F <---
134872024-02-07T19:00:26.185Z 16 320-339 122 122 122 214 214 214 F F F
134882024-02-07T19:00:26.185Z 17 340-359 120 120 120 213 213 213 F F F
134892024-02-07T19:00:26.185Z 18 360-379 122 122 122 214 214 214 F F F
134902024-02-07T19:00:26.185Z 19 380-399 120 120 120 213 213 213 F F F
134912024-02-07T19:00:26.185Z 20 400-419 120 120 120 213 213 213 F F F
134922024-02-07T19:00:26.185Z 21 420-439 118 118 118 210 210 210 T T F <---
134932024-02-07T19:00:26.185Z 22 440-459 122 122 122 214 214 214 F F F
134942024-02-07T19:00:26.185Z 23 460-479 106 106 106 186 186 186 F F F
134952024-02-07T19:00:26.186Z 24 480-499 108 108 108 191 191 191 T T F <---
134962024-02-07T19:00:26.186Z 25 500-519 120 120 120 213 213 213 T T F <---
134972024-02-07T19:00:26.186Z 26 520-539 120 120 120 213 213 213 F F F
134982024-02-07T19:00:26.186Z 27 540-559 120 120 120 213 213 213 F F F
134992024-02-07T19:00:26.186Z 28 560-579 118 118 118 211 211 211 F F F
135002024-02-07T19:00:26.186Z 29 580-599 118 118 118 211 211 211 T T F <---
135012024-02-07T19:00:26.186ZMax gen: 122, Max flush: 215
135022024-02-07T19:00:26.186ZError: Difference in extent metadata found!
135032024-02-07T19:00:26.195ZA Difference in extent metadata is expected here
135042024-02-07T19:00:26.196ZOn loop 61
135052024-02-07T19:00:26.196Z
135062024-02-07T19:00:26.196ZVerifying data now
135072024-02-07T19:00:31.785ZNo change after verify
135082024-02-07T19:00:31.785ZLoop: 61 Downstairs dump after verify (and repair):
135092024-02-07T19:00:31.801ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135102024-02-07T19:00:31.801Z 0 000-019 120 120 120 213 213 213 F F F
135112024-02-07T19:00:31.801Z 1 020-039 119 119 119 212 212 212 F F F
135122024-02-07T19:00:31.802Z 2 040-059 118 118 118 207 207 207 F F F
135132024-02-07T19:00:31.802Z 3 060-079 108 108 108 191 191 191 F F F
135142024-02-07T19:00:31.802Z 4 080-099 114 114 114 201 201 201 F F F
135152024-02-07T19:00:31.802Z 5 100-119 123 123 123 216 216 216 F F F
135162024-02-07T19:00:31.802Z 6 120-139 123 123 123 216 216 216 F F F
135172024-02-07T19:00:31.807Z 7 140-159 123 123 123 216 216 216 F F F
135182024-02-07T19:00:31.807Z 8 160-179 119 119 119 212 212 212 F F F
135192024-02-07T19:00:31.807Z 9 180-199 118 118 118 208 208 208 F F F
135202024-02-07T19:00:31.808Z 10 200-219 117 117 117 206 206 206 F F F
135212024-02-07T19:00:31.808Z 11 220-239 122 122 122 215 215 215 F F F
135222024-02-07T19:00:31.808Z 12 240-259 122 122 122 215 215 215 F F F
135232024-02-07T19:00:31.808Z 13 260-279 122 122 122 215 215 215 F F F
135242024-02-07T19:00:31.808Z 14 280-299 122 122 122 215 215 215 F F F
135252024-02-07T19:00:31.808Z 15 300-319 123 123 123 216 216 216 F F F
135262024-02-07T19:00:31.809Z 16 320-339 122 122 122 214 214 214 F F F
135272024-02-07T19:00:31.809Z 17 340-359 120 120 120 213 213 213 F F F
135282024-02-07T19:00:31.809Z 18 360-379 122 122 122 214 214 214 F F F
135292024-02-07T19:00:31.809Z 19 380-399 120 120 120 213 213 213 F F F
135302024-02-07T19:00:31.809Z 20 400-419 120 120 120 213 213 213 F F F
135312024-02-07T19:00:31.809Z 21 420-439 123 123 123 216 216 216 F F F
135322024-02-07T19:00:31.809Z 22 440-459 122 122 122 214 214 214 F F F
135332024-02-07T19:00:31.810Z 23 460-479 106 106 106 186 186 186 F F F
135342024-02-07T19:00:31.810Z 24 480-499 123 123 123 216 216 216 F F F
135352024-02-07T19:00:31.810Z 25 500-519 123 123 123 216 216 216 F F F
135362024-02-07T19:00:31.810Z 26 520-539 120 120 120 213 213 213 F F F
135372024-02-07T19:00:31.810Z 27 540-559 120 120 120 213 213 213 F F F
135382024-02-07T19:00:31.810Z 28 560-579 118 118 118 211 211 211 F F F
135392024-02-07T19:00:31.811Z 29 580-599 123 123 123 216 216 216 F F F
135402024-02-07T19:00:31.811ZMax gen: 123, Max flush: 216
135412024-02-07T19:00:31.811Z
135422024-02-07T19:00:31.811ZBegin loop 62
135432024-02-07T19:00:31.811ZDownstairs to restart: 0
135442024-02-07T19:00:31.822Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.042681004Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631}
135452024-02-07T19:00:31.822Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.042910363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631}
135462024-02-07T19:00:31.822Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.042930201Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631}
135472024-02-07T19:00:31.828Z{"msg":"Crucible 134a915c-49fe-4d8f-bdf0-c5ee04413bc5 has session id: 99a4dc28-ba6b-4bd2-b57b-02f4876282eb","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.042963506Z","hostname":"Crucible runtime is spawned
135482024-02-07T19:00:31.828Zip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135492024-02-07T19:00:31.828Z{"msg":"Upstairs opts: Upstairs UUID: 134a915c-49fe-4d8f-bdf0-c5ee04413bc5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043053744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135502024-02-07T19:00:31.829Z{"msg":"Crucible stats registered with UUID: 134a915c-49fe-4d8f-bdf0-c5ee04413bc5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043133203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135512024-02-07T19:00:31.829Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043169208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631}
135522024-02-07T19:00:31.829Z{"msg":"134a915c-49fe-4d8f-bdf0-c5ee04413bc5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043211832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135532024-02-07T19:00:31.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043257516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135542024-02-07T19:00:31.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043307419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135552024-02-07T19:00:31.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043356993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135562024-02-07T19:00:31.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043407686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135572024-02-07T19:00:31.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043468978Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135582024-02-07T19:00:31.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043519111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135592024-02-07T19:00:31.830Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043551386Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135602024-02-07T19:00:31.830Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043630606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135612024-02-07T19:00:31.831Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043692758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135622024-02-07T19:00:31.831Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:31.043729482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135632024-02-07T19:00:31.831Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:31.043782155Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135642024-02-07T19:00:31.831Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043811411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135652024-02-07T19:00:31.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043847857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135662024-02-07T19:00:31.832Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:31.043878032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135672024-02-07T19:00:31.832Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043920457Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135682024-02-07T19:00:31.832Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.043958622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135692024-02-07T19:00:31.832Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.044008165Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135702024-02-07T19:00:31.832Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.044368086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135712024-02-07T19:00:31.832Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.044405521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135722024-02-07T19:00:31.832Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.044427668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135732024-02-07T19:00:31.832Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.044499928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135742024-02-07T19:00:31.832Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045024537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135752024-02-07T19:00:31.832Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045063482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135762024-02-07T19:00:31.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045647783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135772024-02-07T19:00:31.832Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045685548Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135782024-02-07T19:00:31.832Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045724812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135792024-02-07T19:00:31.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.04574629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135802024-02-07T19:00:31.832Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045761238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135812024-02-07T19:00:31.832Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:31.045774926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135822024-02-07T19:00:32.825Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04552282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135832024-02-07T19:00:32.825Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04567068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135842024-02-07T19:00:32.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.046807476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135852024-02-07T19:00:32.831Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.046843501Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135862024-02-07T19:00:32.831Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04736901Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135872024-02-07T19:00:32.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.0479592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135882024-02-07T19:00:32.831Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.047986636Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135892024-02-07T19:00:32.831Z{"msg":"[0]R flush_numbers[0..12]: [213, 212, 207, 191, 201, 216, 216, 216, 212, 208, 206, 215]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048002244Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135902024-02-07T19:00:32.831Z{"msg":"[0]R generation[0..12]: [120, 119, 118, 108, 114, 123, 123, 123, 119, 118, 117, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04803483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135912024-02-07T19:00:32.831Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048072205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135922024-02-07T19:00:32.832Z{"msg":"[1]R flush_numbers[0..12]: [213, 212, 207, 191, 201, 216, 216, 216, 212, 208, 206, 215]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04810588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135932024-02-07T19:00:32.832Z{"msg":"[1]R generation[0..12]: [120, 119, 118, 108, 114, 123, 123, 123, 119, 118, 117, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048139956Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135942024-02-07T19:00:32.832Z{Wait for a query_work_queue command to finish before sending IO
135952024-02-07T19:00:32.832Z"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048167152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135962024-02-07T19:00:32.832Z{"msg":"[2]R flush_numbers[0..12]: [213, 212, 207, 191, 201, 216, 216, 216, 212, 208, 206, 215]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048195128Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135972024-02-07T19:00:32.832Z{"msg":"[2]R generation[0..12]: [120, 119, 118, 108, 114, 123, 123, 123, 119, 118, 117, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048213816Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135982024-02-07T19:00:32.832Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048244481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
135992024-02-07T19:00:32.832Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048262049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136002024-02-07T19:00:32.832Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048292405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136012024-02-07T19:00:32.832Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048308473Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136022024-02-07T19:00:32.832Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048338689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136032024-02-07T19:00:32.832Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048356186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136042024-02-07T19:00:32.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048384773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136052024-02-07T19:00:32.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04840146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136062024-02-07T19:00:32.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048432266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136072024-02-07T19:00:32.832Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048453573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136082024-02-07T19:00:32.832Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.04848108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136092024-02-07T19:00:32.832Z{"msg":"134a915c-49fe-4d8f-bdf0-c5ee04413bc5 is now active with session: 99a4dc28-ba6b-4bd2-b57b-02f4876282eb","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048499207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136102024-02-07T19:00:32.832Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048526143Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631,"session_id":"99a4dc28-ba6b-4bd2-b57b-02f4876282eb"}
136112024-02-07T19:00:32.832Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:32.048540832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631}
136122024-02-07T19:00:32.832ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136132024-02-07T19:00:32.832ZLoading write count information from file "/tmp/test_repair_verify.data"
136142024-02-07T19:00:32.832ZRun Repair workload
136152024-02-07T19:00:32.833Z01/30 Write block 38 len 16384 data: 7 5 7 7
136162024-02-07T19:00:32.837Z02/30 Write block 128 len 32768 data: 12 12 10 7 7 8 7 5
136172024-02-07T19:00:32.837Z03/30 Write block 562 len 20480 data: 9 9 12 12 12
136182024-02-07T19:00:32.837Z04/30 Read block 366 len 8192
136192024-02-07T19:00:32.837Z05/30 Flush
136202024-02-07T19:00:32.837Z06/30 Write block 103 len 8192 data: 8 13
136212024-02-07T19:00:32.837Z07/30 Read block 248 len 40960
136222024-02-07T19:00:32.837Z08/30 Flush
136232024-02-07T19:00:32.840Z09/30 Write block 328 len 12288 data: 7 6 8
136242024-02-07T19:00:32.840Z10/30 Write block 546 len 24576 data: 6 10 11 13 12 9
136252024-02-07T19:00:32.841Z11/30 Read block 14 len 8192
136262024-02-07T19:00:32.841Z12/30 Write block 215 len 40960 data: 6 5 5 6 6 7 7 6 7 8
136272024-02-07T19:00:32.844Z13/30 Write block 119 len 28672 data: 14 13 11 13 12 11 11
136282024-02-07T19:00:32.844Z14/30 Read block 423 len 16384
136292024-02-07T19:00:32.844Z15/30 Write block 247 len 32768 data: 10 10 9 8 7 8 8 8
136302024-02-07T19:00:32.844Z16/30 Read block 356 len 32768
136312024-02-07T19:00:32.847Z17/30 Read block 73 len 28672
136322024-02-07T19:00:32.848Z18/30 Write block 450 len 24576 data: 10 11 9 11 10 9
136332024-02-07T19:00:32.848Z19/30 Write block 29 len 16384 data: 15 14 15 13
136342024-02-07T19:00:32.853Z20/30 Write block 102 len 36864 data: 7 9 14 15 14 14 13 12 10
136352024-02-07T19:00:32.853Z21/30 Write block 46 len 32768 data: 6 7 8 7 8 11 9 10
136362024-02-07T19:00:32.854Z22/30 Write block 190 len 24576 data: 7 11 12 10 11 9
136372024-02-07T19:00:32.854Z23/30 Write block 325 len 24576 data: 7 6 5 8 7 9
136382024-02-07T19:00:32.854Z24/30 Read block 576 len 8192
136392024-02-07T19:00:32.854Z25/30 Flush
136402024-02-07T19:00:32.865Z26/30 Flush
136412024-02-07T19:00:32.866Z27/30 Write block 123 len 20480 data: 13 12 12 11 11
136422024-02-07T19:00:32.866Z28/30 Write block 369 len 8192 data: 12 15
136432024-02-07T19:00:32.866Z29/30 Write block 235 len 12288 data: 19 18 19
136442024-02-07T19:00:32.866Z30/30 Write block 464 len 8192 data: 7 8
136452024-02-07T19:00:32.866Z----------------------------------------------------------------
136462024-02-07T19:00:32.866Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
136472024-02-07T19:00:32.866ZStates: Active Active Active
136482024-02-07T19:00:32.866ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136492024-02-07T19:00:32.866Z 1 Acked 1000 Write 4 Done Done Done false
136502024-02-07T19:00:32.866Z 2 Acked 1001 Write 8 Done Done Done false
136512024-02-07T19:00:32.866Z 3 Acked 1002 Write 5 Done Done Done false
136522024-02-07T19:00:32.867Z 4 Acked 1003 Read 2 Sent Done Done false
136532024-02-07T19:00:32.867Z 5 Acked 1004 Flush 0 Sent Done Done false
136542024-02-07T19:00:32.867Z 6 Acked 1005 Write 2 Sent Done Done false
136552024-02-07T19:00:32.867Z 7 Acked 1006 Read 10 Sent Done Done false
136562024-02-07T19:00:32.867Z 8 Acked 1007 Flush 0 Sent Done Done false
136572024-02-07T19:00:32.868Z 9 Acked 1008 Write 3 Sent Done Done false
136582024-02-07T19:00:32.868Z 10 Acked 1009 Write 6 Sent Done Done false
136592024-02-07T19:00:32.868Z 11 Acked 1010 Read 2 Sent Done Done false
136602024-02-07T19:00:32.868Z 12 Acked 1011 Write 10 Sent Done Done false
136612024-02-07T19:00:32.868Z 13 Acked 1012 Write 7 Sent Done Done false
136622024-02-07T19:00:32.868Z 14 Acked 1013 Read 4 Sent Done Done false
136632024-02-07T19:00:32.868Z 15 Acked 1014 Write 8 Sent Done Done false
136642024-02-07T19:00:32.868Z 16 Acked 1015 Read 8 Sent Done Done false
136652024-02-07T19:00:32.868Z 17 Acked 1016 Read 7 Sent Done Done false
136662024-02-07T19:00:32.868Z 18 Acked 1017 Write 6 Sent Done Done false
136672024-02-07T19:00:32.868Z 19 Acked 1018 Write 4 Sent Done Done false
136682024-02-07T19:00:32.869Z 20 Acked 1019 Write 9 Sent Done Done false
136692024-02-07T19:00:32.869Z 21 Acked 1020 Write 8 Sent Done Done false
136702024-02-07T19:00:32.869Z 22 Acked 1021 Write 6 Sent Done Done false
136712024-02-07T19:00:32.869Z 23 Acked 1022 Write 6 Sent Done Done false
136722024-02-07T19:00:32.869Z 24 Acked 1023 Read 2 Sent Done Done false
136732024-02-07T19:00:32.869Z 25 Acked 1024 Flush 0 Sent Done Done false
136742024-02-07T19:00:32.869Z 26 Acked 1025 Flush 0 Sent Done Done false
136752024-02-07T19:00:32.869Z 27 Acked 1026 Write 5 Sent Done Done false
136762024-02-07T19:00:32.869Z 28 Acked 1027 Write 2 Sent Done Done false
136772024-02-07T19:00:32.870Z 29 Acked 1028 Write 3 Sent Sent Sent false
136782024-02-07T19:00:32.870Z 30 Acked 1029 Write 2 Sent Sent Sent false
136792024-02-07T19:00:32.870Z STATES DS:0 DS:1 DS:2 TOTAL
136802024-02-07T19:00:32.870Z New 0 0 0 0
136812024-02-07T19:00:32.870Z Sent 27 2 2 31
136822024-02-07T19:00:32.870Z Done 3 28 28 59
136832024-02-07T19:00:32.871Z Skipped 0 0 0 0
136842024-02-07T19:00:32.871Z Error 0 0 0 0
136852024-02-07T19:00:32.871ZLast Flush: 0 1025 1025
136862024-02-07T19:00:32.871ZDownstairs last five completed:
136872024-02-07T19:00:32.871ZUpstairs last five completed: 30 29 28 27 26
136882024-02-07T19:00:32.881Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:32.079032708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1631}
136892024-02-07T19:00:32.881ZWrote out file "/tmp/test_repair_verify.data"
136902024-02-07T19:00:32.881Z
136912024-02-07T19:00:32.881ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136922024-02-07T19:00:32.889ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136932024-02-07T19:00:32.889Z 0 000-019 120 120 120 213 213 213 F F F
136942024-02-07T19:00:32.889Z 1 020-039 119 124 124 212 219 219 T F F <---
136952024-02-07T19:00:32.890Z 2 040-059 118 124 124 207 219 219 T F F <---
136962024-02-07T19:00:32.890Z 3 060-079 108 108 108 191 191 191 F F F
136972024-02-07T19:00:32.890Z 4 080-099 114 114 114 201 201 201 F F F
136982024-02-07T19:00:32.890Z 5 100-119 123 124 124 216 219 219 F F F <---
136992024-02-07T19:00:32.890Z 6 120-139 123 124 124 216 219 219 T T T <---
137002024-02-07T19:00:32.890Z 7 140-159 123 123 123 216 216 216 F F F
137012024-02-07T19:00:32.890Z 8 160-179 119 119 119 212 212 212 F F F
137022024-02-07T19:00:32.891Z 9 180-199 118 124 124 208 219 219 F F F <---
137032024-02-07T19:00:32.891Z 10 200-219 117 124 124 206 219 219 F F F <---
137042024-02-07T19:00:32.891Z 11 220-239 122 124 124 215 219 219 F T T <---
137052024-02-07T19:00:32.891Z 12 240-259 122 124 124 215 219 219 F F F <---
137062024-02-07T19:00:32.891Z 13 260-279 122 122 122 215 215 215 F F F
137072024-02-07T19:00:32.891Z 14 280-299 122 122 122 215 215 215 F F F
137082024-02-07T19:00:32.891Z 15 300-319 123 123 123 216 216 216 F F F
137092024-02-07T19:00:32.891Z 16 320-339 122 124 124 214 219 219 F F F <---
137102024-02-07T19:00:32.892Z 17 340-359 120 120 120 213 213 213 F F F
137112024-02-07T19:00:32.892Z 18 360-379 122 122 122 214 214 214 F T T <---
137122024-02-07T19:00:32.892Z 19 380-399 120 120 120 213 213 213 F F F
137132024-02-07T19:00:32.892Z 20 400-419 120 120 120 213 213 213 F F F
137142024-02-07T19:00:32.892Z 21 420-439 123 123 123 216 216 216 F F F
137152024-02-07T19:00:32.892Z 22 440-459 122 124 124 214 219 219 F F F <---
137162024-02-07T19:00:32.893Z 23 460-479 106 106 106 186 186 186 F T T <---
137172024-02-07T19:00:32.893Z 24 480-499 123 123 123 216 216 216 F F F
137182024-02-07T19:00:32.893Z 25 500-519 123 123 123 216 216 216 F F F
137192024-02-07T19:00:32.893Z 26 520-539 120 120 120 213 213 213 F F F
137202024-02-07T19:00:32.893Z 27 540-559 120 124 124 213 219 219 F F F <---
137212024-02-07T19:00:32.893Z 28 560-579 118 124 124 211 217 217 T F F <---
137222024-02-07T19:00:32.894Z 29 580-599 123 123 123 216 216 216 F F F
137232024-02-07T19:00:32.894ZMax gen: 124, Max flush: 219
137242024-02-07T19:00:32.894ZError: Difference in extent metadata found!
137252024-02-07T19:00:32.913ZA Difference in extent metadata is expected here
137262024-02-07T19:00:32.913ZOn loop 62
137272024-02-07T19:00:32.913Z
137282024-02-07T19:00:32.913ZVerifying data now
137292024-02-07T19:00:38.983ZNo change after verify
137302024-02-07T19:00:38.984ZLoop: 62 Downstairs dump after verify (and repair):
137312024-02-07T19:00:38.994ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137322024-02-07T19:00:38.994Z 0 000-019 120 120 120 213 213 213 F F F
137332024-02-07T19:00:38.994Z 1 020-039 124 124 124 219 219 219 F F F
137342024-02-07T19:00:38.994Z 2 040-059 124 124 124 219 219 219 F F F
137352024-02-07T19:00:38.994Z 3 060-079 108 108 108 191 191 191 F F F
137362024-02-07T19:00:38.994Z 4 080-099 114 114 114 201 201 201 F F F
137372024-02-07T19:00:38.994Z 5 100-119 124 124 124 219 219 219 F F F
137382024-02-07T19:00:38.994Z 6 120-139 125 125 125 220 220 220 F F F
137392024-02-07T19:00:38.994Z 7 140-159 123 123 123 216 216 216 F F F
137402024-02-07T19:00:38.994Z 8 160-179 119 119 119 212 212 212 F F F
137412024-02-07T19:00:38.994Z 9 180-199 124 124 124 219 219 219 F F F
137422024-02-07T19:00:38.994Z 10 200-219 124 124 124 219 219 219 F F F
137432024-02-07T19:00:38.995Z 11 220-239 125 125 125 220 220 220 F F F
137442024-02-07T19:00:38.995Z 12 240-259 124 124 124 219 219 219 F F F
137452024-02-07T19:00:38.995Z 13 260-279 122 122 122 215 215 215 F F F
137462024-02-07T19:00:38.995Z 14 280-299 122 122 122 215 215 215 F F F
137472024-02-07T19:00:38.995Z 15 300-319 123 123 123 216 216 216 F F F
137482024-02-07T19:00:38.995Z 16 320-339 124 124 124 219 219 219 F F F
137492024-02-07T19:00:38.995Z 17 340-359 120 120 120 213 213 213 F F F
137502024-02-07T19:00:38.995Z 18 360-379 125 125 125 220 220 220 F F F
137512024-02-07T19:00:38.995Z 19 380-399 120 120 120 213 213 213 F F F
137522024-02-07T19:00:38.995Z 20 400-419 120 120 120 213 213 213 F F F
137532024-02-07T19:00:38.995Z 21 420-439 123 123 123 216 216 216 F F F
137542024-02-07T19:00:38.995Z 22 440-459 124 124 124 219 219 219 F F F
137552024-02-07T19:00:38.995Z 23 460-479 125 125 125 220 220 220 F F F
137562024-02-07T19:00:38.996Z 24 480-499 123 123 123 216 216 216 F F F
137572024-02-07T19:00:38.996Z 25 500-519 123 123 123 216 216 216 F F F
137582024-02-07T19:00:38.996Z 26 520-539 120 120 120 213 213 213 F F F
137592024-02-07T19:00:38.996Z 27 540-559 124 124 124 219 219 219 F F F
137602024-02-07T19:00:38.996Z 28 560-579 124 124 124 217 217 217 F F F
137612024-02-07T19:00:38.996Z 29 580-599 123 123 123 216 216 216 F F F
137622024-02-07T19:00:38.996ZMax gen: 125, Max flush: 220
137632024-02-07T19:00:38.996Z
137642024-02-07T19:00:38.996ZBegin loop 63
137652024-02-07T19:00:38.996ZDownstairs to restart: 1
137662024-02-07T19:00:38.996Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.817797126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639}
137672024-02-07T19:00:38.996Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818035514Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639}
137682024-02-07T19:00:38.996Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818053571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639}
137692024-02-07T19:00:38.997Z{"msg":"Crucible 93ed824a-0669-42f0-91b6-2b07c8cb74f9 has session id: 14c32f12-c820-4901-bd55-9d7e2a113cb7","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818087627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137702024-02-07T19:00:38.997Z{"msg":"Upstairs opts: Upstairs UUID: 93ed824a-0669-42f0-91b6-2b07c8cb74f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
137712024-02-07T19:00:38.997Z,"v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818143819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137722024-02-07T19:00:38.997Z{"msg":"Crucible stats registered with UUID: 93ed824a-0669-42f0-91b6-2b07c8cb74f9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818171475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137732024-02-07T19:00:38.997Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818203021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639}
137742024-02-07T19:00:38.997Z{"msg":"93ed824a-0669-42f0-91b6-2b07c8cb74f9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818316546Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137752024-02-07T19:00:38.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818364349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137762024-02-07T19:00:38.997Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818392945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137772024-02-07T19:00:38.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.81843162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137782024-02-07T19:00:38.997Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818489832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137792024-02-07T19:00:38.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818536316Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137802024-02-07T19:00:38.998Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818561243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137812024-02-07T19:00:38.998Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818680757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137822024-02-07T19:00:38.998Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818739959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137832024-02-07T19:00:38.998Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818757106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137842024-02-07T19:00:38.998Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818771664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137852024-02-07T19:00:38.998Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.81879979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137862024-02-07T19:00:38.998Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.818820878Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137872024-02-07T19:00:38.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.819307042Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137882024-02-07T19:00:38.998Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.819346367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137892024-02-07T19:00:38.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.819371943Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137902024-02-07T19:00:38.998Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.819406368Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137912024-02-07T19:00:38.999Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.819973852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137922024-02-07T19:00:38.999Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820027604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137932024-02-07T19:00:38.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820634162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137942024-02-07T19:00:38.999Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820675607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137952024-02-07T19:00:38.999Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820701263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137962024-02-07T19:00:38.999Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820808169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137972024-02-07T19:00:38.999Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820860802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137982024-02-07T19:00:38.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820904026Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
137992024-02-07T19:00:38.999Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820922054Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138002024-02-07T19:00:38.999Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.820956499Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138012024-02-07T19:00:38.999Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.821418876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138022024-02-07T19:00:38.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.82190892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138032024-02-07T19:00:39.000Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.821949195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138042024-02-07T19:00:39.000Z{"msg":"[0]R flush_numbers[0..12]: [213, 219, 219, 191, 201, 219, 220, 216, 212, 219, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.821967362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138052024-02-07T19:00:39.000Z{"msg":"[0]R generation[0..12]: [120, 124, 124, 108, 114, 124, 125, 123, 119, 124, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.8219827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138062024-02-07T19:00:39.000Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822013106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138072024-02-07T19:00:39.000Z{"msg":"[1]R flush_numbers[0..12]: [213, 219, 219, 191, 201, 219, 220, 216, 212, 219, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822030453Z","hostname":"Wait for a query_work_queue command to finish before sending IO
138082024-02-07T19:00:39.000Zip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138092024-02-07T19:00:39.000Z{"msg":"[1]R generation[0..12]: [120, 124, 124, 108, 114, 124, 125, 123, 119, 124, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822049871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138102024-02-07T19:00:39.000Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822065189Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138112024-02-07T19:00:39.000Z{"msg":"[2]R flush_numbers[0..12]: [213, 219, 219, 191, 201, 219, 220, 216, 212, 219, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822088656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138122024-02-07T19:00:39.000Z{"msg":"[2]R generation[0..12]: [120, 124, 124, 108, 114, 124, 125, 123, 119, 124, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822103384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138132024-02-07T19:00:39.000Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822118182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138142024-02-07T19:00:39.000Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822139069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138152024-02-07T19:00:39.000Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822155787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138162024-02-07T19:00:39.001Z{"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822189152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138172024-02-07T19:00:39.001Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822210689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138182024-02-07T19:00:39.002Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822236556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138192024-02-07T19:00:39.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822267521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138202024-02-07T19:00:39.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822282919Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138212024-02-07T19:00:39.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822306636Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138222024-02-07T19:00:39.003Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822335422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138232024-02-07T19:00:39.003Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.82234973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138242024-02-07T19:00:39.003Z{"msg":"93ed824a-0669-42f0-91b6-2b07c8cb74f9 is now active with session: 14c32f12-c820-4901-bd55-9d7e2a113cb7","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822374437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138252024-02-07T19:00:39.003Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822389685Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639,"session_id":"14c32f12-c820-4901-bd55-9d7e2a113cb7"}
138262024-02-07T19:00:39.003Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:37.822413332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639}
138272024-02-07T19:00:39.003ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138282024-02-07T19:00:39.003ZLoading write count information from file "/tmp/test_repair_verify.data"
138292024-02-07T19:00:39.003ZRun Repair workload
138302024-02-07T19:00:39.003Z01/30 Write block 483 len 40960 data: 10 9 7 7 8 8 6 7 7 7
138312024-02-07T19:00:39.010Z02/30 Read block 234 len 20480
138322024-02-07T19:00:39.011Z03/30 Read block 19 len 24576
138332024-02-07T19:00:39.011Z04/30 Read block 125 len 8192
138342024-02-07T19:00:39.011Z05/30 Read block 10 len 36864
138352024-02-07T19:00:39.011Z06/30 Write block 440 len 12288 data: 11 11 12
138362024-02-07T19:00:39.011Z07/30 Write block 103 len 28672 data: 10 15 16 15 15 14 13
138372024-02-07T19:00:39.011Z08/30 Read block 0 len 8192
138382024-02-07T19:00:39.012Z09/30 Write block 192 len 32768 data: 13 11 12 10 9 10 12 9
138392024-02-07T19:00:39.012Z10/30 Write block 399 len 8192 data: 12 13
138402024-02-07T19:00:39.012Z11/30 Write block 95 len 8192 data: 13 13
138412024-02-07T19:00:39.012Z12/30 Write block 385 len 4096 data: 9
138422024-02-07T19:00:39.012Z13/30 Read block 360 len 24576
138432024-02-07T19:00:39.012Z14/30 Flush
138442024-02-07T19:00:39.012Z15/30 Flush
138452024-02-07T19:00:39.012Z16/30 Flush
138462024-02-07T19:00:39.013Z17/30 Write block 382 len 28672 data: 7 10 9 10 10 10 12
138472024-02-07T19:00:39.013Z18/30 Write block 293 len 24576 data: 10 8 9 8 10 10
138482024-02-07T19:00:39.013Z19/30 Read block 357 len 16384
138492024-02-07T19:00:39.013Z20/30 Write block 74 len 32768 data: 5 5 6 8 10 9 7 5
138502024-02-07T19:00:39.013Z21/30 Write block 124 len 12288 data: 13 13 12
138512024-02-07T19:00:39.013Z22/30 Write block 335 len 40960 data: 8 9 9 11 13 10 8 8 8 9
138522024-02-07T19:00:39.013Z23/30 Read block 129 len 20480
138532024-02-07T19:00:39.014Z24/30 Write block 260 len 20480 data: 10 9 11 14 15
138542024-02-07T19:00:39.014Z25/30 Read block 317 len 12288
138552024-02-07T19:00:39.014Z26/30 Write block 535 len 32768 data: 12 12 13 11 11 12 13 13
138562024-02-07T19:00:39.014Z27/30 Write block 299 len 20480 data: 7 7 7 9 5
138572024-02-07T19:00:39.014Z28/30 Write block 156 len 24576 data: 8 6 8 7 7 8
138582024-02-07T19:00:39.014Z29/30 Read block 558 len 36864
138592024-02-07T19:00:39.014Z30/30 Read block 334 len 20480
138602024-02-07T19:00:39.014Z----------------------------------------------------------------
138612024-02-07T19:00:39.014Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
138622024-02-07T19:00:39.015ZStates: Active Active Active
138632024-02-07T19:00:39.015ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138642024-02-07T19:00:39.015Z 1 Acked 1000 Write 10 Done Sent Done false
138652024-02-07T19:00:39.015Z 2 Acked 1001 Read 5 Done Sent Done false
138662024-02-07T19:00:39.015Z 3 Acked 1002 Read 6 Done Sent Done false
138672024-02-07T19:00:39.015Z 4 Acked 1003 Read 2 Done Sent Done false
138682024-02-07T19:00:39.015Z 5 Acked 1004 Read 9 Done Sent Done false
138692024-02-07T19:00:39.015Z 6 Acked 1005 Write 3 Done Sent Done false
138702024-02-07T19:00:39.016Z 7 Acked 1006 Write 7 Done Sent Done false
138712024-02-07T19:00:39.016Z 8 Acked 1007 Read 2 Done Sent Done false
138722024-02-07T19:00:39.016Z 9 Acked 1008 Write 8 Done Sent Done false
138732024-02-07T19:00:39.016Z 10 Acked 1009 Write 2 Done Sent Done false
138742024-02-07T19:00:39.016Z 11 Acked 1010 Write 2 Done Sent Done false
138752024-02-07T19:00:39.016Z 12 Acked 1011 Write 1 Done Sent Done false
138762024-02-07T19:00:39.016Z 13 Acked 1012 Read 6 Done Sent Done false
138772024-02-07T19:00:39.016Z 14 Acked 1013 Flush 0 Done Sent Done false
138782024-02-07T19:00:39.016Z 15 Acked 1014 Flush 0 Done Sent Done false
138792024-02-07T19:00:39.016Z 16 Acked 1015 Flush 0 Done Sent Done false
138802024-02-07T19:00:39.016Z 17 Acked 1016 Write 7 Done Sent Done false
138812024-02-07T19:00:39.016Z 18 Acked 1017 Write 6 Done Sent Done false
138822024-02-07T19:00:39.016Z 19 Acked 1018 Read 4 Done Sent Done false
138832024-02-07T19:00:39.016Z 20 Acked 1019 Write 8 Done Sent Done false
138842024-02-07T19:00:39.016Z 21 Acked 1020 Write 3 Done Sent Done false
138852024-02-07T19:00:39.017Z 22 Acked 1021 Write 10 Done Sent Done false
138862024-02-07T19:00:39.017Z 23 Acked 1022 Read 5 Done Sent Done false
138872024-02-07T19:00:39.017Z 24 Acked 1023 Write 5 Done Sent Done false
138882024-02-07T19:00:39.017Z 25 Acked 1024 Read 3 Done Sent Done false
138892024-02-07T19:00:39.017Z 26 Acked 1025 Write 8 Done Sent Done false
138902024-02-07T19:00:39.017Z 27 Acked 1026 Write 5 Done Sent Done false
138912024-02-07T19:00:39.017Z 28 Acked 1027 Write 6 Done Sent Done false
138922024-02-07T19:00:39.017Z 29 Acked 1028 Read 9 Done Sent Done false
138932024-02-07T19:00:39.017Z 30 Acked 1029 Read 5 Done Sent Done false
138942024-02-07T19:00:39.017Z STATES DS:0 DS:1 DS:2 TOTAL
138952024-02-07T19:00:39.017Z New 0 0 0 0
138962024-02-07T19:00:39.017Z Sent 0 30 0 30
138972024-02-07T19:00:39.017Z Done 30 0 30 60
138982024-02-07T19:00:39.017Z Skipped 0 0 0 0
138992024-02-07T19:00:39.017Z Error 0 0 0 0
139002024-02-07T19:00:39.017ZLast Flush: 1015 0 1015
139012024-02-07T19:00:39.017ZDownstairs last five completed:
139022024-02-07T19:00:39.017ZUpstairs last five completed: 30 29 28 27 26
139032024-02-07T19:00:39.017Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:37.850828679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1639}
139042024-02-07T19:00:39.017ZWrote out file "/tmp/test_repair_verify.data"
139052024-02-07T19:00:39.017Z
139062024-02-07T19:00:39.017ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
139072024-02-07T19:00:39.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139082024-02-07T19:00:39.018Z 0 000-019 120 120 120 213 213 213 F F F
139092024-02-07T19:00:39.018Z 1 020-039 124 124 124 219 219 219 F F F
139102024-02-07T19:00:39.018Z 2 040-059 124 124 124 219 219 219 F F F
139112024-02-07T19:00:39.018Z 3 060-079 108 108 108 191 191 191 T F T <---
139122024-02-07T19:00:39.018Z 4 080-099 126 114 126 221 201 221 T F T <---
139132024-02-07T19:00:39.018Z 5 100-119 126 124 126 221 219 221 F F F <---
139142024-02-07T19:00:39.018Z 6 120-139 125 125 125 220 220 220 T F T <---
139152024-02-07T19:00:39.018Z 7 140-159 123 123 123 216 216 216 T F T <---
139162024-02-07T19:00:39.018Z 8 160-179 119 119 119 212 212 212 T F T <---
139172024-02-07T19:00:39.018Z 9 180-199 126 124 126 221 219 221 F F F <---
139182024-02-07T19:00:39.018Z 10 200-219 124 124 124 219 219 219 F F F
139192024-02-07T19:00:39.018Z 11 220-239 125 125 125 220 220 220 F F F
139202024-02-07T19:00:39.018Z 12 240-259 124 124 124 219 219 219 F F F
139212024-02-07T19:00:39.018Z 13 260-279 122 122 122 215 215 215 T F T <---
139222024-02-07T19:00:39.018Z 14 280-299 122 122 122 215 215 215 T F T <---
139232024-02-07T19:00:39.018Z 15 300-319 123 123 123 216 216 216 T F T <---
139242024-02-07T19:00:39.018Z 16 320-339 124 124 124 219 219 219 T F T <---
139252024-02-07T19:00:39.018Z 17 340-359 120 120 120 213 213 213 T F T <---
139262024-02-07T19:00:39.018Z 18 360-379 125 125 125 220 220 220 F F F
139272024-02-07T19:00:39.018Z 19 380-399 126 120 126 221 213 221 T F T <---
139282024-02-07T19:00:39.018Z 20 400-419 126 120 126 221 213 221 F F F <---
139292024-02-07T19:00:39.018Z 21 420-439 123 123 123 216 216 216 F F F
139302024-02-07T19:00:39.018Z 22 440-459 126 124 126 221 219 221 F F F <---
139312024-02-07T19:00:39.027Z 23 460-479 125 125 125 220 220 220 F F F
139322024-02-07T19:00:39.027Z 24 480-499 126 123 126 221 216 221 F F F <---
139332024-02-07T19:00:39.027Z 25 500-519 123 123 123 216 216 216 F F F
139342024-02-07T19:00:39.027Z 26 520-539 120 120 120 213 213 213 T F T <---
139352024-02-07T19:00:39.028Z 27 540-559 124 124 124 219 219 219 T F T <---
139362024-02-07T19:00:39.028Z 28 560-579 124 124 124 217 217 217 F F F
139372024-02-07T19:00:39.028Z 29 580-599 123 123 123 216 216 216 F F F
139382024-02-07T19:00:39.028ZMax gen: 126, Max flush: 221
139392024-02-07T19:00:39.028ZError: Difference in extent metadata found!
139402024-02-07T19:00:39.028ZA Difference in extent metadata is expected here
139412024-02-07T19:00:39.028ZOn loop 63
139422024-02-07T19:00:39.028Z
139432024-02-07T19:00:39.029ZVerifying data now
139442024-02-07T19:00:43.560ZNo change after verify
139452024-02-07T19:00:43.560ZLoop: 63 Downstairs dump after verify (and repair):
139462024-02-07T19:00:43.575ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139472024-02-07T19:00:43.575Z 0 000-019 120 120 120 213 213 213 F F F
139482024-02-07T19:00:43.575Z 1 020-039 124 124 124 219 219 219 F F F
139492024-02-07T19:00:43.575Z 2 040-059 124 124 124 219 219 219 F F F
139502024-02-07T19:00:43.575Z 3 060-079 127 127 127 222 222 222 F F F
139512024-02-07T19:00:43.576Z 4 080-099 127 127 127 222 222 222 F F F
139522024-02-07T19:00:43.576Z 5 100-119 126 126 126 221 221 221 F F F
139532024-02-07T19:00:43.576Z 6 120-139 127 127 127 222 222 222 F F F
139542024-02-07T19:00:43.576Z 7 140-159 127 127 127 222 222 222 F F F
139552024-02-07T19:00:43.576Z 8 160-179 127 127 127 222 222 222 F F F
139562024-02-07T19:00:43.576Z 9 180-199 126 126 126 221 221 221 F F F
139572024-02-07T19:00:43.576Z 10 200-219 124 124 124 219 219 219 F F F
139582024-02-07T19:00:43.576Z 11 220-239 125 125 125 220 220 220 F F F
139592024-02-07T19:00:43.576Z 12 240-259 124 124 124 219 219 219 F F F
139602024-02-07T19:00:43.577Z 13 260-279 127 127 127 222 222 222 F F F
139612024-02-07T19:00:43.577Z 14 280-299 127 127 127 222 222 222 F F F
139622024-02-07T19:00:43.577Z 15 300-319 127 127 127 222 222 222 F F F
139632024-02-07T19:00:43.577Z 16 320-339 127 127 127 222 222 222 F F F
139642024-02-07T19:00:43.577Z 17 340-359 127 127 127 222 222 222 F F F
139652024-02-07T19:00:43.577Z 18 360-379 125 125 125 220 220 220 F F F
139662024-02-07T19:00:43.577Z 19 380-399 127 127 127 222 222 222 F F F
139672024-02-07T19:00:43.577Z 20 400-419 126 126 126 221 221 221 F F F
139682024-02-07T19:00:43.577Z 21 420-439 123 123 123 216 216 216 F F F
139692024-02-07T19:00:43.578Z 22 440-459 126 126 126 221 221 221 F F F
139702024-02-07T19:00:43.578Z 23 460-479 125 125 125 220 220 220 F F F
139712024-02-07T19:00:43.578Z 24 480-499 126 126 126 221 221 221 F F F
139722024-02-07T19:00:43.578Z 25 500-519 123 123 123 216 216 216 F F F
139732024-02-07T19:00:43.578Z 26 520-539 127 127 127 222 222 222 F F F
139742024-02-07T19:00:43.578Z 27 540-559 127 127 127 222 222 222 F F F
139752024-02-07T19:00:43.578Z 28 560-579 124 124 124 217 217 217 F F F
139762024-02-07T19:00:43.578Z 29 580-599 123 123 123 216 216 216 F F F
139772024-02-07T19:00:43.578ZMax gen: 127, Max flush: 222
139782024-02-07T19:00:43.578Z
139792024-02-07T19:00:43.578ZBegin loop 64
139802024-02-07T19:00:43.578ZDownstairs to restart: 2
139812024-02-07T19:00:43.578Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.751952559Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647}
139822024-02-07T19:00:43.579Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752258088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647}
139832024-02-07T19:00:43.579Z{Crucible runtime is spawned
139842024-02-07T19:00:43.579Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752301282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647}
139852024-02-07T19:00:43.579Z{"msg":"Crucible 39c52ade-9d0b-41a1-99e6-ba727ef4065c has session id: d8e66158-d524-40ae-b2e0-005c08758ed4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752345206Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139862024-02-07T19:00:43.579Z{"msg":"Upstairs opts: Upstairs UUID: 39c52ade-9d0b-41a1-99e6-ba727ef4065c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.75238967Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139872024-02-07T19:00:43.579Z{"msg":"Crucible stats registered with UUID: 39c52ade-9d0b-41a1-99e6-ba727ef4065c","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752429145Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139882024-02-07T19:00:43.579Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752453322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647}
139892024-02-07T19:00:43.579Z{"msg":"39c52ade-9d0b-41a1-99e6-ba727ef4065c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752511354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139902024-02-07T19:00:43.579Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.75254127Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139912024-02-07T19:00:43.579Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752560037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139922024-02-07T19:00:43.579Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752588884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139932024-02-07T19:00:43.579Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752619989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139942024-02-07T19:00:43.579Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752665993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139952024-02-07T19:00:43.580Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752701118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139962024-02-07T19:00:43.580Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752814653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139972024-02-07T19:00:43.580Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752889683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139982024-02-07T19:00:43.580Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.752952204Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
139992024-02-07T19:00:43.580Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.75298839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140002024-02-07T19:00:43.580Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753032833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140012024-02-07T19:00:43.580Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:42.753074008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140022024-02-07T19:00:43.580Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:42.753115652Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140032024-02-07T19:00:43.580Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:42.753237356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140042024-02-07T19:00:43.580Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753269232Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140052024-02-07T19:00:43.580Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753292918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140062024-02-07T19:00:43.580Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753332963Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140072024-02-07T19:00:43.580Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753694934Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140082024-02-07T19:00:43.581Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753737078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140092024-02-07T19:00:43.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753765724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140102024-02-07T19:00:43.581Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.753814888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140112024-02-07T19:00:43.581Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.75438477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140122024-02-07T19:00:43.581Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.754413387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140132024-02-07T19:00:43.581Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.754942135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140142024-02-07T19:00:43.581Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.754988369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140152024-02-07T19:00:43.581Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.755014385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140162024-02-07T19:00:43.581Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.755058899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140172024-02-07T19:00:43.581Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.755095984Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140182024-02-07T19:00:43.581Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:42.755136529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140192024-02-07T19:00:44.537Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.755081615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140202024-02-07T19:00:44.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.755152376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140212024-02-07T19:00:44.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.756012059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140222024-02-07T19:00:44.544Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.756048805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140232024-02-07T19:00:44.544Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.756588741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140242024-02-07T19:00:44.544Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757195159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140252024-02-07T19:00:44.544Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757240933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140262024-02-07T19:00:44.544Z{"msg":"[0]R flush_numbers[0..12]: [213, 219, 219, 222, 222, 221, 222, 222, 222, 221, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757290057Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140272024-02-07T19:00:44.544Z{"msg":"[0]R generation[0..12]: [120, 124, 124, 127, 127, 126, 127, 127, 127, 126, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757329411Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140282024-02-07T19:00:44.545Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757356358Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140292024-02-07T19:00:44.545Z{"msg":"[1]R flush_numbers[0..12]: [213, 219, 219, 222, 222, 221, 222, 222, 222, 221, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757374925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140302024-02-07T19:00:44.545Z{"msg":"[1]R generation[0..12]: [120, 124, 124, 127, 127, 126, 127, 127, 127, 126, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757408021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140312024-02-07T19:00:44.545Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757435307Z","hostname":"Wait for a query_work_queue command to finish before sending IO
140322024-02-07T19:00:44.545Zip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140332024-02-07T19:00:44.545Z{"msg":"[2]R flush_numbers[0..12]: [213, 219, 219, 222, 222, 221, 222, 222, 222, 221, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757470842Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140342024-02-07T19:00:44.545Z{"msg":"[2]R generation[0..12]: [120, 124, 124, 127, 127, 126, 127, 127, 127, 126, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757510417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140352024-02-07T19:00:44.546Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757529914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140362024-02-07T19:00:44.546Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.75756127Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140372024-02-07T19:00:44.546Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757598135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140382024-02-07T19:00:44.546Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757618732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140392024-02-07T19:00:44.546Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757649868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140402024-02-07T19:00:44.546Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757672005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140412024-02-07T19:00:44.546Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757693042Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140422024-02-07T19:00:44.546Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.75771136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140432024-02-07T19:00:44.546Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757750944Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140442024-02-07T19:00:44.547Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757807247Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140452024-02-07T19:00:44.547Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140462024-02-07T19:00:44.547Z,"time":"2024-02-07T19:00:43.757849081Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140472024-02-07T19:00:44.547Z{"msg":"39c52ade-9d0b-41a1-99e6-ba727ef4065c is now active with session: d8e66158-d524-40ae-b2e0-005c08758ed4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757879887Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140482024-02-07T19:00:44.547Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757907543Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647,"session_id":"d8e66158-d524-40ae-b2e0-005c08758ed4"}
140492024-02-07T19:00:44.547Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:43.757923951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647}
140502024-02-07T19:00:44.547ZLoading write count information from file "/tmp/test_repair_verify.data"
140512024-02-07T19:00:44.548ZRun Repair workload
140522024-02-07T19:00:44.548Z01/30 Flush
140532024-02-07T19:00:44.548Z02/30 Write block 588 len 32768 data: 12 11 11 13 12 10 9 7
140542024-02-07T19:00:44.555Z03/30 Read block 230 len 32768
140552024-02-07T19:00:44.555Z04/30 Write block 565 len 32768 data: 13 13 13 13 12 12 10 7
140562024-02-07T19:00:44.555Z05/30 Read block 458 len 24576
140572024-02-07T19:00:44.555Z06/30 Write block 148 len 16384 data: 15 18 12 12
140582024-02-07T19:00:44.555Z07/30 Read block 303 len 8192
140592024-02-07T19:00:44.556Z08/30 Write block 584 len 12288 data: 10 11 11
140602024-02-07T19:00:44.556Z09/30 Read block 363 len 12288
140612024-02-07T19:00:44.556Z10/30 Read block 391 len 16384
140622024-02-07T19:00:44.556Z11/30 Write block 39 len 16384 data: 6 8 8 7
140632024-02-07T19:00:44.556Z12/30 Write block 134 len 8192 data: 8 6
140642024-02-07T19:00:44.556Z13/30 Read block 66 len 36864
140652024-02-07T19:00:44.557Z14/30 Read block 252 len 4096
140662024-02-07T19:00:44.557Z15/30 Read block 68 len 36864
140672024-02-07T19:00:44.557Z16/30 Write block 34 len 20480 data: 14 13 13 9 8
140682024-02-07T19:00:44.557Z17/30 Read block 49 len 12288
140692024-02-07T19:00:44.557Z18/30 Read block 269 len 20480
140702024-02-07T19:00:44.561Z19/30 Read block 335 len 40960
140712024-02-07T19:00:44.561Z20/30 Read block 459 len 24576
140722024-02-07T19:00:44.562Z21/30 Write block 124 len 24576 data: 14 14 13 12 13 13
140732024-02-07T19:00:44.562Z22/30 Read block 419 len 12288
140742024-02-07T19:00:44.562Z23/30 Write block 543 len 36864 data: 7 6 5 7 11 12 14 13 10
140752024-02-07T19:00:44.562Z24/30 Read block 516 len 40960
140762024-02-07T19:00:44.577Z25/30 Write block 187 len 32768 data: 7 8 8 8 12 14 12 13
140772024-02-07T19:00:44.577Z26/30 Write block 567 len 8192 data: 14 14
140782024-02-07T19:00:44.577Z27/30 Read block 342 len 16384
140792024-02-07T19:00:44.577Z28/30 Read block 267 len 24576
140802024-02-07T19:00:44.578Z29/30 Read block 475 len 12288
140812024-02-07T19:00:44.578Z30/30 Read block 238 len 32768
140822024-02-07T19:00:44.582Z----------------------------------------------------------------
140832024-02-07T19:00:44.582Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
140842024-02-07T19:00:44.582ZStates: Active Active Active
140852024-02-07T19:00:44.582ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140862024-02-07T19:00:44.582Z 1 Acked 1000 Flush 0 Done Done Sent false
140872024-02-07T19:00:44.582Z 2 Acked 1001 Write 8 Done Done Sent false
140882024-02-07T19:00:44.582Z 3 Acked 1002 Read 8 Done Done Sent false
140892024-02-07T19:00:44.583Z 4 Acked 1003 Write 8 Done Done Sent false
140902024-02-07T19:00:44.583Z 5 Acked 1004 Read 6 Done Done Sent false
140912024-02-07T19:00:44.583Z 6 Acked 1005 Write 4 Done Done Sent false
140922024-02-07T19:00:44.583Z 7 Acked 1006 Read 2 Done Done Sent false
140932024-02-07T19:00:44.583Z 8 Acked 1007 Write 3 Done Done Sent false
140942024-02-07T19:00:44.583Z 9 Acked 1008 Read 3 Done Done Sent false
140952024-02-07T19:00:44.583Z 10 Acked 1009 Read 4 Done Done Sent false
140962024-02-07T19:00:44.583Z 11 Acked 1010 Write 4 Done Done Sent false
140972024-02-07T19:00:44.584Z 12 Acked 1011 Write 2 Done Done Sent false
140982024-02-07T19:00:44.584Z 13 Acked 1012 Read 9 Done Done Sent false
140992024-02-07T19:00:44.584Z 14 Acked 1013 Read 1 Done Done Sent false
141002024-02-07T19:00:44.584Z 15 Acked 1014 Read 9 Done Done Sent false
141012024-02-07T19:00:44.584Z 16 Acked 1015 Write 5 Done Done Sent false
141022024-02-07T19:00:44.584Z 17 Acked 1016 Read 3 Done Done Sent false
141032024-02-07T19:00:44.584Z 18 Acked 1017 Read 5 Done Done Sent false
141042024-02-07T19:00:44.584Z 19 Acked 1018 Read 10 Done Done Sent false
141052024-02-07T19:00:44.584Z 20 Acked 1019 Read 6 Done Done Sent false
141062024-02-07T19:00:44.584Z 21 Acked 1020 Write 6 Done Done Sent false
141072024-02-07T19:00:44.584Z 22 Acked 1021 Read 3 Done Done Sent false
141082024-02-07T19:00:44.585Z 23 Acked 1022 Write 9 Done Done Sent false
141092024-02-07T19:00:44.585Z 24 Acked 1023 Read 10 Done Done Sent false
141102024-02-07T19:00:44.585Z 25 Acked 1024 Write 8 Done Done Sent false
141112024-02-07T19:00:44.585Z 26 Acked 1025 Write 2 Done Done Sent false
141122024-02-07T19:00:44.585Z 27 Acked 1026 Read 4 Done Done Sent false
141132024-02-07T19:00:44.585Z 28 Acked 1027 Read 6 Done Done Sent false
141142024-02-07T19:00:44.585Z 29 Acked 1028 Read 3 Done Done Sent false
141152024-02-07T19:00:44.585Z 30 Acked 1029 Read 8 Done Done Sent false
141162024-02-07T19:00:44.585Z STATES DS:0 DS:1 DS:2 TOTAL
141172024-02-07T19:00:44.586Z New 0 0 0 0
141182024-02-07T19:00:44.586Z Sent 0 0 30 30
141192024-02-07T19:00:44.586Z Done 30 30 0 60
141202024-02-07T19:00:44.586Z Skipped 0 0 0 0
141212024-02-07T19:00:44.586Z Error 0 0 0 0
141222024-02-07T19:00:44.586ZLast Flush: 1000 1000 0
141232024-02-07T19:00:44.586ZDownstairs last five completed:
141242024-02-07T19:00:44.586ZUpstairs last five completed: 30 29 28 27 26
141252024-02-07T19:00:44.586Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:43.785648001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1647}
141262024-02-07T19:00:44.586ZWrote out file "/tmp/test_repair_verify.data"
141272024-02-07T19:00:44.586Z
141282024-02-07T19:00:44.587ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141292024-02-07T19:00:44.593ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141302024-02-07T19:00:44.594Z 0 000-019 120 120 120 213 213 213 F F F
141312024-02-07T19:00:44.594Z 1 020-039 124 124 124 219 219 219 T T F <---
141322024-02-07T19:00:44.594Z 2 040-059 124 124 124 219 219 219 T T F <---
141332024-02-07T19:00:44.598Z 3 060-079 127 127 127 222 222 222 F F F
141342024-02-07T19:00:44.598Z 4 080-099 127 127 127 222 222 222 F F F
141352024-02-07T19:00:44.598Z 5 100-119 126 126 126 221 221 221 F F F
141362024-02-07T19:00:44.598Z 6 120-139 127 127 127 222 222 222 T T F <---
141372024-02-07T19:00:44.599Z 7 140-159 127 127 127 222 222 222 T T F <---
141382024-02-07T19:00:44.599Z 8 160-179 127 127 127 222 222 222 F F F
141392024-02-07T19:00:44.599Z 9 180-199 126 126 126 221 221 221 T T F <---
141402024-02-07T19:00:44.599Z 10 200-219 124 124 124 219 219 219 F F F
141412024-02-07T19:00:44.599Z 11 220-239 125 125 125 220 220 220 F F F
141422024-02-07T19:00:44.599Z 12 240-259 124 124 124 219 219 219 F F F
141432024-02-07T19:00:44.599Z 13 260-279 127 127 127 222 222 222 F F F
141442024-02-07T19:00:44.599Z 14 280-299 127 127 127 222 222 222 F F F
141452024-02-07T19:00:44.600Z 15 300-319 127 127 127 222 222 222 F F F
141462024-02-07T19:00:44.600Z 16 320-339 127 127 127 222 222 222 F F F
141472024-02-07T19:00:44.600Z 17 340-359 127 127 127 222 222 222 F F F
141482024-02-07T19:00:44.600Z 18 360-379 125 125 125 220 220 220 F F F
141492024-02-07T19:00:44.600Z 19 380-399 127 127 127 222 222 222 F F F
141502024-02-07T19:00:44.600Z 20 400-419 126 126 126 221 221 221 F F F
141512024-02-07T19:00:44.600Z 21 420-439 123 123 123 216 216 216 F F F
141522024-02-07T19:00:44.601Z 22 440-459 126 126 126 221 221 221 F F F
141532024-02-07T19:00:44.601Z 23 460-479 125 125 125 220 220 220 F F F
141542024-02-07T19:00:44.601Z 24 480-499 126 126 126 221 221 221 F F F
141552024-02-07T19:00:44.601Z 25 500-519 123 123 123 216 216 216 F F F
141562024-02-07T19:00:44.601Z 26 520-539 127 127 127 222 222 222 F F F
141572024-02-07T19:00:44.601Z 27 540-559 127 127 127 222 222 222 T T F <---
141582024-02-07T19:00:44.602Z 28 560-579 124 124 124 217 217 217 T T F <---
141592024-02-07T19:00:44.602Z 29 580-599 123 123 123 216 216 216 T T F <---
141602024-02-07T19:00:44.602ZMax gen: 127, Max flush: 222
141612024-02-07T19:00:44.602ZError: Difference in extent metadata found!
141622024-02-07T19:00:44.602ZA Difference in extent metadata is expected here
141632024-02-07T19:00:44.602ZOn loop 64
141642024-02-07T19:00:44.602Z
141652024-02-07T19:00:44.607ZVerifying data now
141662024-02-07T19:00:49.030ZNo change after verify
141672024-02-07T19:00:49.030ZLoop: 64 Downstairs dump after verify (and repair):
141682024-02-07T19:00:49.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141692024-02-07T19:00:49.045Z 0 000-019 120 120 120 213 213 213 F F F
141702024-02-07T19:00:49.045Z 1 020-039 128 128 128 223 223 223 F F F
141712024-02-07T19:00:49.045Z 2 040-059 128 128 128 223 223 223 F F F
141722024-02-07T19:00:49.046Z 3 060-079 127 127 127 222 222 222 F F F
141732024-02-07T19:00:49.046Z 4 080-099 127 127 127 222 222 222 F F F
141742024-02-07T19:00:49.046Z 5 100-119 126 126 126 221 221 221 F F F
141752024-02-07T19:00:49.046Z 6 120-139 128 128 128 223 223 223 F F F
141762024-02-07T19:00:49.046Z 7 140-159 128 128 128 223 223 223 F F F
141772024-02-07T19:00:49.046Z 8 160-179 127 127 127 222 222 222 F F F
141782024-02-07T19:00:49.055Z 9 180-199 128 128 128 223 223 223 F F F
141792024-02-07T19:00:49.055Z 10 200-219 124 124 124 219 219 219 F F F
141802024-02-07T19:00:49.055Z 11 220-239 125 125 125 220 220 220 F F F
141812024-02-07T19:00:49.056Z 12 240-259 124 124 124 219 219 219 F F F
141822024-02-07T19:00:49.056Z 13 260-279 127 127 127 222 222 222 F F F
141832024-02-07T19:00:49.056Z 14 280-299 127 127 127 222 222 222 F F F
141842024-02-07T19:00:49.056Z 15 300-319 127 127 127 222 222 222 F F F
141852024-02-07T19:00:49.056Z 16 320-339 127 127 127 222 222 222 F F F
141862024-02-07T19:00:49.056Z 17 340-359 127 127 127 222 222 222 F F F
141872024-02-07T19:00:49.056Z 18 360-379 125 125 125 220 220 220 F F F
141882024-02-07T19:00:49.056Z 19 380-399 127 127 127 222 222 222 F F F
141892024-02-07T19:00:49.057Z 20 400-419 126 126 126 221 221 221 F F F
141902024-02-07T19:00:49.057Z 21 420-439 123 123 123 216 216 216 F F F
141912024-02-07T19:00:49.057Z 22 440-459 126 126 126 221 221 221 F F F
141922024-02-07T19:00:49.057Z 23 460-479 125 125 125 220 220 220 F F F
141932024-02-07T19:00:49.057Z 24 480-499 126 126 126 221 221 221 F F F
141942024-02-07T19:00:49.057Z 25 500-519 123 123 123 216 216 216 F F F
141952024-02-07T19:00:49.057Z 26 520-539 127 127 127 222 222 222 F F F
141962024-02-07T19:00:49.057Z 27 540-559 128 128 128 223 223 223 F F F
141972024-02-07T19:00:49.057Z 28 560-579 128 128 128 223 223 223 F F F
141982024-02-07T19:00:49.058Z 29 580-599 128 128 128 223 223 223 F F F
141992024-02-07T19:00:49.058ZMax gen: 128, Max flush: 223
142002024-02-07T19:00:49.058Z
142012024-02-07T19:00:49.058ZBegin loop 65
142022024-02-07T19:00:49.058ZDownstairs to restart: 2
142032024-02-07T19:00:49.066Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.283671962Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655}
142042024-02-07T19:00:49.073Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.283946114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655}Crucible runtime is spawned
142052024-02-07T19:00:49.073Z
142062024-02-07T19:00:49.073Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284006106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655}
142072024-02-07T19:00:49.073Z{"msg":"Crucible 33bd6050-c744-4fc5-8a1c-719d92a5cbee has session id: 8dd39f82-a0ed-48b1-a707-7813c7613792","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284036192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142082024-02-07T19:00:49.073Z{"msg":"Upstairs opts: Upstairs UUID: 33bd6050-c744-4fc5-8a1c-719d92a5cbee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284092005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142092024-02-07T19:00:49.074Z{"msg":"Crucible stats registered with UUID: 33bd6050-c744-4fc5-8a1c-719d92a5cbee","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.28412809Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142102024-02-07T19:00:49.074Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284145458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655}
142112024-02-07T19:00:49.074Z{"msg":"33bd6050-c744-4fc5-8a1c-719d92a5cbee active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284225317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142122024-02-07T19:00:49.074Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284268041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142132024-02-07T19:00:49.074Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284295387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142142024-02-07T19:00:49.074Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284337481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142152024-02-07T19:00:49.074Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284374087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142162024-02-07T19:00:49.074Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284401673Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142172024-02-07T19:00:49.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284429579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142182024-02-07T19:00:49.075Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284469694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142192024-02-07T19:00:49.075Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.28450051Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142202024-02-07T19:00:49.075Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284525466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142212024-02-07T19:00:49.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284550573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142222024-02-07T19:00:49.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284575569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142232024-02-07T19:00:49.076Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:48.284626962Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142242024-02-07T19:00:49.076Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:48.284658358Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142252024-02-07T19:00:49.076Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:48.284681105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142262024-02-07T19:00:49.076Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284699203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142272024-02-07T19:00:49.076Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284733328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142282024-02-07T19:00:49.076Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.284775452Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142292024-02-07T19:00:49.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.28523496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142302024-02-07T19:00:49.077Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.285271115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142312024-02-07T19:00:49.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.285288683Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142322024-02-07T19:00:49.077Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.285322168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142332024-02-07T19:00:49.077Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.285873864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142342024-02-07T19:00:49.077Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.286027653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142352024-02-07T19:00:49.077Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.286397093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142362024-02-07T19:00:49.077Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.286444267Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142372024-02-07T19:00:49.078Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.286498809Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142382024-02-07T19:00:49.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.286542313Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142392024-02-07T19:00:49.078Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.28656976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142402024-02-07T19:00:49.078Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:48.286601885Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142412024-02-07T19:00:50.068Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.285235759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142422024-02-07T19:00:50.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.285454909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142432024-02-07T19:00:50.074Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.286645518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142442024-02-07T19:00:50.074Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.286688823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142452024-02-07T19:00:50.074Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.287252126Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142462024-02-07T19:00:50.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.287869343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142472024-02-07T19:00:50.074Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.287920276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142482024-02-07T19:00:50.074Z{"msg":"[0]R flush_numbers[0..12]: [213, 223, 223, 222, 222, 221, 223, 223, 222, 223, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.287941743Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142492024-02-07T19:00:50.074Z{"msg":"[0]R generation[0..12]: [120, 128, 128, 127, 127, 126, 128, 128, 127, 128, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.287969029Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142502024-02-07T19:00:50.074Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.287987547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142512024-02-07T19:00:50.074Z{"msg":"[1]R flush_numbers[0..12]: [213, 223, 223, 222, 222, 221, 223, 223, 222, 223, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.28803973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142522024-02-07T19:00:50.074Z{"msg":"[1]R generation[0..12]: [120, 128, 128, 127, 127, 126, 128, 128, 127, 128, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288058787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142532024-02-07T19:00:50.074Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288081514Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142542024-02-07T19:00:50.074Z{"msg":"[2]R flush_numbers[0..12]: [213, 223, 223, 222, 222, 221, 223, 223, 222, 223, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288099412Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142552024-02-07T19:00:50.074ZWait for a query_work_queue command to finish before sending IO
142562024-02-07T19:00:50.074Z{"msg":"[2]R generation[0..12]: [120, 128, 128, 127, 127, 126, 128, 128, 127, 128, 124, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288129108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142572024-02-07T19:00:50.074Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288148465Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142582024-02-07T19:00:50.074Z{"msg":"Max found gen is 129","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288174782Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142592024-02-07T19:00:50.074Z{"msg":"Generation requested: 130 >= found:129","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288208407Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142602024-02-07T19:00:50.074Z{"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288235213Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142612024-02-07T19:00:50.074Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288266579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142622024-02-07T19:00:50.074Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288288306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142632024-02-07T19:00:50.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288318252Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142642024-02-07T19:00:50.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288342149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142652024-02-07T19:00:50.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288371965Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142662024-02-07T19:00:50.075Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288397041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142672024-02-07T19:00:50.075Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288423248Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142682024-02-07T19:00:50.075Z{"msg":"33bd6050-c744-4fc5-8a1c-719d92a5cbee is now active with session: 8dd39f82-a0ed-48b1-a707-7813c7613792","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288447835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142692024-02-07T19:00:50.075Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288473401Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655,"session_id":"8dd39f82-a0ed-48b1-a707-7813c7613792"}
142702024-02-07T19:00:50.075Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:49.288496688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655}
142712024-02-07T19:00:50.075ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142722024-02-07T19:00:50.075ZLoading write count information from file "/tmp/test_repair_verify.data"
142732024-02-07T19:00:50.075ZRun Repair workload
142742024-02-07T19:00:50.075Z01/30 Write block 263 len 24576 data: 15 16 13 13 14 10
142752024-02-07T19:00:50.079Z02/30 Read block 287 len 40960
142762024-02-07T19:00:50.079Z03/30 Write block 444 len 16384 data: 13 14 12 12
142772024-02-07T19:00:50.079Z04/30 Write block 32 len 32768 data: 14 13 15 14 14 10 9 7
142782024-02-07T19:00:50.079Z05/30 Write block 169 len 8192 data: 14 13
142792024-02-07T19:00:50.079Z06/30 Read block 116 len 24576
142802024-02-07T19:00:50.079Z07/30 Write block 321 len 4096 data: 7
142812024-02-07T19:00:50.079Z08/30 Write block 488 len 28672 data: 9 7 8 8 8 8 11
142822024-02-07T19:00:50.079Z09/30 Read block 97 len 4096
142832024-02-07T19:00:50.079Z10/30 Flush
142842024-02-07T19:00:50.083Z11/30 Flush
142852024-02-07T19:00:50.083Z12/30 Write block 111 len 24576 data: 12 10 9 7 8 10
142862024-02-07T19:00:50.083Z13/30 Flush
142872024-02-07T19:00:50.086Z14/30 Write block 339 len 20480 data: 14 11 9 9 9
142882024-02-07T19:00:50.086Z15/30 Read block 322 len 24576
142892024-02-07T19:00:50.090Z16/30 Read block 123 len 36864
142902024-02-07T19:00:50.090Z17/30 Write block 578 len 20480 data: 6 7 8 10 11
142912024-02-07T19:00:50.090Z18/30 Read block 164 len 20480
142922024-02-07T19:00:50.090Z19/30 Read block 501 len 16384
142932024-02-07T19:00:50.093Z20/30 Read block 110 len 32768
142942024-02-07T19:00:50.093Z21/30 Read block 243 len 40960
142952024-02-07T19:00:50.093Z22/30 Read block 75 len 20480
142962024-02-07T19:00:50.093Z23/30 Read block 178 len 36864
142972024-02-07T19:00:50.096Z24/30 Write block 223 len 32768 data: 8 9 7 5 5 6 8 7
142982024-02-07T19:00:50.096Z25/30 Flush
142992024-02-07T19:00:50.100Z26/30 Write block 175 len 20480 data: 8 7 8 8 8
143002024-02-07T19:00:50.100Z27/30 Read block 500 len 4096
143012024-02-07T19:00:50.100Z28/30 Write block 524 len 36864 data: 8 7 8 10 9 10 9 7 8
143022024-02-07T19:00:50.100Z29/30 Write block 559 len 4096 data: 11
143032024-02-07T19:00:50.103Z30/30 Write block 498 len 24576 data: 9 8 9 7 10 12
143042024-02-07T19:00:50.103Z----------------------------------------------------------------
143052024-02-07T19:00:50.103Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
143062024-02-07T19:00:50.104ZStates: Active Active Active
143072024-02-07T19:00:50.104ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143082024-02-07T19:00:50.104Z 1 Acked 1000 Write 6 Done Done Done false
143092024-02-07T19:00:50.104Z 2 Acked 1001 Read 10 Done Done Done false
143102024-02-07T19:00:50.104Z 3 Acked 1002 Write 4 Done Done Done false
143112024-02-07T19:00:50.104Z 4 Acked 1003 Write 8 Done Done Sent false
143122024-02-07T19:00:50.104Z 5 Acked 1004 Write 2 Done Done Sent false
143132024-02-07T19:00:50.104Z 6 Acked 1005 Read 6 Done Done Sent false
143142024-02-07T19:00:50.104Z 7 Acked 1006 Write 1 Done Done Sent false
143152024-02-07T19:00:50.104Z 8 Acked 1007 Write 7 Done Done Sent false
143162024-02-07T19:00:50.104Z 9 Acked 1008 Read 1 Done Done Sent false
143172024-02-07T19:00:50.104Z 10 Acked 1009 Flush 0 Done Done Sent false
143182024-02-07T19:00:50.104Z 11 Acked 1010 Flush 0 Done Done Sent false
143192024-02-07T19:00:50.104Z 12 Acked 1011 Write 6 Done Done Sent false
143202024-02-07T19:00:50.104Z 13 Acked 1012 Flush 0 Done Done Sent false
143212024-02-07T19:00:50.104Z 14 Acked 1013 Write 5 Done Done Sent false
143222024-02-07T19:00:50.104Z 15 Acked 1014 Read 6 Done Done Sent false
143232024-02-07T19:00:50.104Z 16 Acked 1015 Read 9 Done Done Sent false
143242024-02-07T19:00:50.104Z 17 Acked 1016 Write 5 Done Done Sent false
143252024-02-07T19:00:50.104Z 18 Acked 1017 Read 5 Done Done Sent false
143262024-02-07T19:00:50.104Z 19 Acked 1018 Read 4 Done Done Sent false
143272024-02-07T19:00:50.104Z 20 Acked 1019 Read 8 Done Done Sent false
143282024-02-07T19:00:50.104Z 21 Acked 1020 Read 10 Done Done Sent false
143292024-02-07T19:00:50.104Z 22 Acked 1021 Read 5 Done Done Sent false
143302024-02-07T19:00:50.105Z 23 Acked 1022 Read 9 Done Done Sent false
143312024-02-07T19:00:50.105Z 24 Acked 1023 Write 8 Done Done Sent false
143322024-02-07T19:00:50.105Z 25 Acked 1024 Flush 0 Done Done Sent false
143332024-02-07T19:00:50.105Z 26 Acked 1025 Write 5 Done Done Sent false
143342024-02-07T19:00:50.105Z 27 Acked 1026 Read 1 Done Done Sent false
143352024-02-07T19:00:50.105Z 28 Acked 1027 Write 9 Done Done Sent false
143362024-02-07T19:00:50.105Z 29 Acked 1028 Write 1 Sent Sent Sent false
143372024-02-07T19:00:50.105Z 30 Acked 1029 Write 6 Sent Sent Sent false
143382024-02-07T19:00:50.105Z STATES DS:0 DS:1 DS:2 TOTAL
143392024-02-07T19:00:50.105Z New 0 0 0 0
143402024-02-07T19:00:50.105Z Sent 2 2 27 31
143412024-02-07T19:00:50.105Z Done 28 28 3 59
143422024-02-07T19:00:50.105Z Skipped 0 0 0 0
143432024-02-07T19:00:50.105Z Error 0 0 0 0
143442024-02-07T19:00:50.105ZLast Flush: 1024 1024 0
143452024-02-07T19:00:50.105ZDownstairs last five completed:
143462024-02-07T19:00:50.105ZUpstairs last five completed: 30 29 28 27 26
143472024-02-07T19:00:50.105Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:49.319371372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1655}
143482024-02-07T19:00:50.105ZWrote out file "/tmp/test_repair_verify.data"
143492024-02-07T19:00:50.109Z
143502024-02-07T19:00:50.109ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143512024-02-07T19:00:50.127ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143522024-02-07T19:00:50.127Z 0 000-019 120 120 120 213 213 213 F F F
143532024-02-07T19:00:50.127Z 1 020-039 130 130 128 224 224 223 F F F <---
143542024-02-07T19:00:50.127Z 2 040-059 128 128 128 223 223 223 F F F
143552024-02-07T19:00:50.127Z 3 060-079 127 127 127 222 222 222 F F F
143562024-02-07T19:00:50.127Z 4 080-099 127 127 127 222 222 222 F F F
143572024-02-07T19:00:50.127Z 5 100-119 130 130 126 226 226 221 F F F <---
143582024-02-07T19:00:50.127Z 6 120-139 128 128 128 223 223 223 F F F
143592024-02-07T19:00:50.127Z 7 140-159 128 128 128 223 223 223 F F F
143602024-02-07T19:00:50.131Z 8 160-179 130 130 127 224 224 222 T T F <---
143612024-02-07T19:00:50.131Z 9 180-199 128 128 128 223 223 223 F F F
143622024-02-07T19:00:50.131Z 10 200-219 124 124 124 219 219 219 F F F
143632024-02-07T19:00:50.131Z 11 220-239 130 130 125 227 227 220 F F F <---
143642024-02-07T19:00:50.131Z 12 240-259 124 124 124 219 219 219 F F F
143652024-02-07T19:00:50.131Z 13 260-279 130 130 127 224 224 222 F F T <---
143662024-02-07T19:00:50.131Z 14 280-299 127 127 127 222 222 222 F F F
143672024-02-07T19:00:50.131Z 15 300-319 127 127 127 222 222 222 F F F
143682024-02-07T19:00:50.131Z 16 320-339 130 130 127 227 227 222 F F F <---
143692024-02-07T19:00:50.131Z 17 340-359 130 130 127 227 227 222 F F F <---
143702024-02-07T19:00:50.131Z 18 360-379 125 125 125 220 220 220 F F F
143712024-02-07T19:00:50.131Z 19 380-399 127 127 127 222 222 222 F F F
143722024-02-07T19:00:50.131Z 20 400-419 126 126 126 221 221 221 F F F
143732024-02-07T19:00:50.131Z 21 420-439 123 123 123 216 216 216 F F F
143742024-02-07T19:00:50.131Z 22 440-459 130 130 126 224 224 221 F F T <---
143752024-02-07T19:00:50.131Z 23 460-479 125 125 125 220 220 220 F F F
143762024-02-07T19:00:50.131Z 24 480-499 130 130 126 224 224 221 T T F <---
143772024-02-07T19:00:50.131Z 25 500-519 123 123 123 216 216 216 T T F <---
143782024-02-07T19:00:50.131Z 26 520-539 127 127 127 222 222 222 T T F <---
143792024-02-07T19:00:50.131Z 27 540-559 128 128 128 223 223 223 T T F <---
143802024-02-07T19:00:50.131Z 28 560-579 130 130 128 227 227 223 F F F <---
143812024-02-07T19:00:50.131Z 29 580-599 130 130 128 227 227 223 F F F <---
143822024-02-07T19:00:50.132ZMax gen: 130, Max flush: 227
143832024-02-07T19:00:50.132ZError: Difference in extent metadata found!
143842024-02-07T19:00:50.132ZA Difference in extent metadata is expected here
143852024-02-07T19:00:50.132ZOn loop 65
143862024-02-07T19:00:50.132Z
143872024-02-07T19:00:50.135ZVerifying data now
143882024-02-07T19:00:55.799ZNo change after verify
143892024-02-07T19:00:55.799ZLoop: 65 Downstairs dump after verify (and repair):
143902024-02-07T19:00:55.813ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143912024-02-07T19:00:55.813Z 0 000-019 120 120 120 213 213 213 F F F
143922024-02-07T19:00:55.813Z 1 020-039 130 130 130 224 224 224 F F F
143932024-02-07T19:00:55.813Z 2 040-059 128 128 128 223 223 223 F F F
143942024-02-07T19:00:55.814Z 3 060-079 127 127 127 222 222 222 F F F
143952024-02-07T19:00:55.814Z 4 080-099 127 127 127 222 222 222 F F F
143962024-02-07T19:00:55.814Z 5 100-119 130 130 130 226 226 226 F F F
143972024-02-07T19:00:55.814Z 6 120-139 128 128 128 223 223 223 F F F
143982024-02-07T19:00:55.814Z 7 140-159 128 128 128 223 223 223 F F F
143992024-02-07T19:00:55.823Z 8 160-179 131 131 131 228 228 228 F F F
144002024-02-07T19:00:55.823Z 9 180-199 128 128 128 223 223 223 F F F
144012024-02-07T19:00:55.824Z 10 200-219 124 124 124 219 219 219 F F F
144022024-02-07T19:00:55.824Z 11 220-239 130 130 130 227 227 227 F F F
144032024-02-07T19:00:55.824Z 12 240-259 124 124 124 219 219 219 F F F
144042024-02-07T19:00:55.824Z 13 260-279 130 130 130 224 224 224 F F F
144052024-02-07T19:00:55.824Z 14 280-299 127 127 127 222 222 222 F F F
144062024-02-07T19:00:55.824Z 15 300-319 127 127 127 222 222 222 F F F
144072024-02-07T19:00:55.824Z 16 320-339 130 130 130 227 227 227 F F F
144082024-02-07T19:00:55.824Z 17 340-359 130 130 130 227 227 227 F F F
144092024-02-07T19:00:55.825Z 18 360-379 125 125 125 220 220 220 F F F
144102024-02-07T19:00:55.825Z 19 380-399 127 127 127 222 222 222 F F F
144112024-02-07T19:00:55.825Z 20 400-419 126 126 126 221 221 221 F F F
144122024-02-07T19:00:55.825Z 21 420-439 123 123 123 216 216 216 F F F
144132024-02-07T19:00:55.825Z 22 440-459 130 130 130 224 224 224 F F F
144142024-02-07T19:00:55.825Z 23 460-479 125 125 125 220 220 220 F F F
144152024-02-07T19:00:55.825Z 24 480-499 131 131 131 228 228 228 F F F
144162024-02-07T19:00:55.826Z 25 500-519 131 131 131 228 228 228 F F F
144172024-02-07T19:00:55.826Z 26 520-539 131 131 131 228 228 228 F F F
144182024-02-07T19:00:55.826Z 27 540-559 131 131 131 228 228 228 F F F
144192024-02-07T19:00:55.826Z 28 560-579 130 130 130 227 227 227 F F F
144202024-02-07T19:00:55.826Z 29 580-599 130 130 130 227 227 227 F F F
144212024-02-07T19:00:55.826ZMax gen: 131, Max flush: 228
144222024-02-07T19:00:55.826Z
144232024-02-07T19:00:55.826ZBegin loop 66
144242024-02-07T19:00:55.827ZDownstairs to restart: 2
144252024-02-07T19:00:55.839Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.048648604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663}
144262024-02-07T19:00:55.839Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.048989298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663}
144272024-02-07T19:00:55.840Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049032843Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663}
144282024-02-07T19:00:55.840Z{"msg":"Crucible c78dbeef-0fee-463c-a480-4e05bb423b87 has session id: 348b728f-03f7-4eea-92b3-935a4cb58b36","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
144292024-02-07T19:00:55.840Z2024-02-07T19:00:55.049061069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144302024-02-07T19:00:55.840Z{"msg":"Upstairs opts: Upstairs UUID: c78dbeef-0fee-463c-a480-4e05bb423b87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049128439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144312024-02-07T19:00:55.840Z{"msg":"Crucible stats registered with UUID: c78dbeef-0fee-463c-a480-4e05bb423b87","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049168134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144322024-02-07T19:00:55.840Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049186222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663}
144332024-02-07T19:00:55.841Z{"msg":"c78dbeef-0fee-463c-a480-4e05bb423b87 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049234125Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144342024-02-07T19:00:55.841Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049260142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144352024-02-07T19:00:55.841Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049284568Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144362024-02-07T19:00:55.841Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049326833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144372024-02-07T19:00:55.841Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049383585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144382024-02-07T19:00:55.842Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049431278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144392024-02-07T19:00:55.842Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049474993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144402024-02-07T19:00:55.842Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049514557Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144412024-02-07T19:00:55.842Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049545893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144422024-02-07T19:00:55.842Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049577089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144432024-02-07T19:00:55.842Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049609494Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144442024-02-07T19:00:55.842Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.04964268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144452024-02-07T19:00:55.842Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.049676195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144462024-02-07T19:00:55.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.050469058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144472024-02-07T19:00:55.871Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.050506383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144482024-02-07T19:00:55.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.05052609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144492024-02-07T19:00:55.871Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.050558966Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144502024-02-07T19:00:55.871Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051061708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144512024-02-07T19:00:55.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051191651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144522024-02-07T19:00:55.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051275719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144532024-02-07T19:00:55.872Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051304995Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144542024-02-07T19:00:55.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051599425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144552024-02-07T19:00:55.872Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051648259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144562024-02-07T19:00:55.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051673175Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144572024-02-07T19:00:55.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051759314Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144582024-02-07T19:00:55.872Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051797238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144592024-02-07T19:00:55.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.051823075Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144602024-02-07T19:00:55.872Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052051654Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144612024-02-07T19:00:55.873Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052545027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144622024-02-07T19:00:55.873Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052583312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144632024-02-07T19:00:55.873Z{"msg":"[0]R flush_numbers[0..12]: [213, 224, 223, 222, 222, 226, 223, 223, 228, 223, 219, 227]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.05260052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144642024-02-07T19:00:55.873Z{"msg":"[0]R generation[0..12]: [120, 130, 128, 127, 127, 130, 128, 128, 131, 128, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052623537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144652024-02-07T19:00:55.873Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052639285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144662024-02-07T19:00:55.873Z{"msg":"[1]R flush_numbers[0..12]: [213, 224, 223, 222, 222, 226, 223, 223, 228, 223, 219, 227]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052660772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144672024-02-07T19:00:55.873Z{"msg":"[1]R generation[0..12]: [120, 130, 128, 127, 127, 130, 128, 128, 131, 128, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.05267547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144682024-02-07T19:00:55.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052702106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144692024-02-07T19:00:55.873Z{"msg":"[2]R flush_numbers[0..12]: [213, 224, 223, 222, 222, 226, 223, 223, 228, 223, 219, 227]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
144702024-02-07T19:00:55.873Z,"time":"2024-02-07T19:00:55.052716754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144712024-02-07T19:00:55.873Z{"msg":"[2]R generation[0..12]: [120, 130, 128, 127, 127, 130, 128, 128, 131, 128, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052731692Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144722024-02-07T19:00:55.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052758899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144732024-02-07T19:00:55.874Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052792624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144742024-02-07T19:00:55.874Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052809522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144752024-02-07T19:00:55.874Z{"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052841567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144762024-02-07T19:00:55.874Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052862554Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144772024-02-07T19:00:55.874Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052891681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144782024-02-07T19:00:55.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052905849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144792024-02-07T19:00:55.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052937444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144802024-02-07T19:00:55.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052955162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144812024-02-07T19:00:55.875Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.052984898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144822024-02-07T19:00:55.875Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.053029262Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144832024-02-07T19:00:55.876Z{"msg":"c78dbeef-0fee-463c-a480-4e05bb423b87 is now active with session: 348b728f-03f7-4eea-92b3-935a4cb58b36","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.053049409Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_idRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144842024-02-07T19:00:55.876Z":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144852024-02-07T19:00:55.876Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.053080085Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663,"session_id":"348b728f-03f7-4eea-92b3-935a4cb58b36"}
144862024-02-07T19:00:55.876Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:00:55.053094813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663}
144872024-02-07T19:00:55.877ZLoading write count information from file "/tmp/test_repair_verify.data"
144882024-02-07T19:00:55.877ZRun Repair workload
144892024-02-07T19:00:55.877Z01/30 Write block 514 len 40960 data: 13 12 11 8 10 9 9 6 5 5
144902024-02-07T19:00:55.877Z02/30 Read block 372 len 12288
144912024-02-07T19:00:55.877Z03/30 Read block 492 len 12288
144922024-02-07T19:00:55.877Z04/30 Read block 322 len 20480
144932024-02-07T19:00:55.878Z05/30 Write block 118 len 8192 data: 13 15
144942024-02-07T19:00:55.878Z06/30 Read block 349 len 16384
144952024-02-07T19:00:55.878Z07/30 Read block 586 len 40960
144962024-02-07T19:00:55.878Z08/30 Read block 404 len 20480
144972024-02-07T19:00:55.879Z09/30 Write block 278 len 4096 data: 6
144982024-02-07T19:00:55.879Z10/30 Read block 175 len 32768
144992024-02-07T19:00:55.879Z11/30 Write block 254 len 12288 data: 9 8 8
145002024-02-07T19:00:55.879Z12/30 Read block 327 len 28672
145012024-02-07T19:00:55.899Z13/30 Read block 190 len 24576
145022024-02-07T19:00:55.899Z14/30 Write block 350 len 28672 data: 13 12 9 8 12 9 6
145032024-02-07T19:00:55.899Z15/30 Read block 220 len 36864
145042024-02-07T19:00:55.900Z16/30 Read block 496 len 28672
145052024-02-07T19:00:55.900Z17/30 Read block 48 len 40960
145062024-02-07T19:00:55.900Z18/30 Read block 389 len 36864
145072024-02-07T19:00:55.900Z19/30 Read block 195 len 20480
145082024-02-07T19:00:55.900Z20/30 Read block 402 len 24576
145092024-02-07T19:00:55.900Z21/30 Write block 4 len 36864 data: 7 11 10 13 15 16 12 12 11
145102024-02-07T19:00:55.900Z22/30 Write block 333 len 12288 data: 11 11 9
145112024-02-07T19:00:55.900Z23/30 Write block 128 len 20480 data: 14 14 11 8 8
145122024-02-07T19:00:55.900Z24/30 Write block 199 len 12288 data: 10 8 8
145132024-02-07T19:00:55.901Z25/30 Flush
145142024-02-07T19:00:55.901Z26/30 Write block 185 len 4096 data: 5
145152024-02-07T19:00:55.901Z27/30 Read block 509 len 32768
145162024-02-07T19:00:55.901Z28/30 Write block 398 len 24576 data: 10 13 14 12 13 9
145172024-02-07T19:00:55.901Z29/30 Write block 97 len 16384 data: 11 10 9 7
145182024-02-07T19:00:55.901Z30/30 Write block 590 len 8192 data: 12 14
145192024-02-07T19:00:55.901Z----------------------------------------------------------------
145202024-02-07T19:00:55.901Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
145212024-02-07T19:00:55.901ZStates: Active Active Active
145222024-02-07T19:00:55.902ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145232024-02-07T19:00:55.902Z 1 Acked 1000 Write 10 Done Done Done false
145242024-02-07T19:00:55.902Z 2 Acked 1001 Read 3 Done Done Done false
145252024-02-07T19:00:55.902Z 3 Acked 1002 Read 3 Done Done Done false
145262024-02-07T19:00:55.902Z 4 Acked 1003 Read 5 Done Done Done false
145272024-02-07T19:00:55.902Z 5 Acked 1004 Write 2 Done Done Done false
145282024-02-07T19:00:55.902Z 6 Acked 1005 Read 4 Done Done Done false
145292024-02-07T19:00:55.902Z 7 Acked 1006 Read 10 Done Done Sent false
145302024-02-07T19:00:55.902Z 8 Acked 1007 Read 5 Done Done Sent false
145312024-02-07T19:00:55.902Z 9 Acked 1008 Write 1 Done Done Sent false
145322024-02-07T19:00:55.903Z 10 Acked 1009 Read 8 Done Done Sent false
145332024-02-07T19:00:55.903Z 11 Acked 1010 Write 3 Done Done Sent false
145342024-02-07T19:00:55.903Z 12 Acked 1011 Read 7 Done Done Sent false
145352024-02-07T19:00:55.903Z 13 Acked 1012 Read 6 Done Done Sent false
145362024-02-07T19:00:55.903Z 14 Acked 1013 Write 7 Done Done Sent false
145372024-02-07T19:00:55.903Z 15 Acked 1014 Read 9 Done Done Sent false
145382024-02-07T19:00:55.903Z 16 Acked 1015 Read 7 Done Done Sent false
145392024-02-07T19:00:55.903Z 17 Acked 1016 Read 10 Done Done Sent false
145402024-02-07T19:00:55.903Z 18 Acked 1017 Read 9 Done Done Sent false
145412024-02-07T19:00:55.903Z 19 Acked 1018 Read 5 Done Done Sent false
145422024-02-07T19:00:55.904Z 20 Acked 1019 Read 6 Done Done Sent false
145432024-02-07T19:00:55.904Z 21 Acked 1020 Write 9 Done Done Sent false
145442024-02-07T19:00:55.904Z 22 Acked 1021 Write 3 Done Done Sent false
145452024-02-07T19:00:55.904Z 23 Acked 1022 Write 5 Done Done Sent false
145462024-02-07T19:00:55.904Z 24 Acked 1023 Write 3 Done Done Sent false
145472024-02-07T19:00:55.904Z 25 Acked 1024 Flush 0 Done Done Sent false
145482024-02-07T19:00:55.904Z 26 Acked 1025 Write 1 Done Done Sent false
145492024-02-07T19:00:55.904Z 27 Acked 1026 Read 8 Done Done Sent false
145502024-02-07T19:00:55.904Z 28 Acked 1027 Write 6 Done Done Sent false
145512024-02-07T19:00:55.904Z 29 Acked 1028 Write 4 Sent Sent Sent false
145522024-02-07T19:00:55.905Z 30 Acked 1029 Write 2 Sent Sent Sent false
145532024-02-07T19:00:55.905Z STATES DS:0 DS:1 DS:2 TOTAL
145542024-02-07T19:00:55.905Z New 0 0 0 0
145552024-02-07T19:00:55.905Z Sent 2 2 24 28
145562024-02-07T19:00:55.905Z Done 28 28 6 62
145572024-02-07T19:00:55.905Z Skipped 0 0 0 0
145582024-02-07T19:00:55.905Z Error 0 0 0 0
145592024-02-07T19:00:55.905ZLast Flush: 1024 1024 0
145602024-02-07T19:00:55.905ZDownstairs last five completed:
145612024-02-07T19:00:55.906ZUpstairs last five completed: 30 29 28 27 26
145622024-02-07T19:00:55.906Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:00:55.082341398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1663}
145632024-02-07T19:00:55.906ZWrote out file "/tmp/test_repair_verify.data"
145642024-02-07T19:00:55.906Z
145652024-02-07T19:00:55.906ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145662024-02-07T19:00:55.912ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145672024-02-07T19:00:55.912Z 0 000-019 132 132 120 229 229 213 F F F <---
145682024-02-07T19:00:55.912Z 1 020-039 130 130 130 224 224 224 F F F
145692024-02-07T19:00:55.912Z 2 040-059 128 128 128 223 223 223 F F F
145702024-02-07T19:00:55.912Z 3 060-079 127 127 127 222 222 222 F F F
145712024-02-07T19:00:55.913Z 4 080-099 127 127 127 222 222 222 T T F <---
145722024-02-07T19:00:55.913Z 5 100-119 132 132 130 229 229 226 T T T <---
145732024-02-07T19:00:55.913Z 6 120-139 132 132 128 229 229 223 F F F <---
145742024-02-07T19:00:55.913Z 7 140-159 128 128 128 223 223 223 F F F
145752024-02-07T19:00:55.913Z 8 160-179 131 131 131 228 228 228 F F F
145762024-02-07T19:00:55.913Z 9 180-199 132 132 128 229 229 223 T T F <---
145772024-02-07T19:00:55.913Z 10 200-219 132 132 124 229 229 219 F F F <---
145782024-02-07T19:00:55.913Z 11 220-239 130 130 130 227 227 227 F F F
145792024-02-07T19:00:55.913Z 12 240-259 132 132 124 229 229 219 F F F <---
145802024-02-07T19:00:55.913Z 13 260-279 132 132 130 229 229 224 F F F <---
145812024-02-07T19:00:55.914Z 14 280-299 127 127 127 222 222 222 F F F
145822024-02-07T19:00:55.914Z 15 300-319 127 127 127 222 222 222 F F F
145832024-02-07T19:00:55.914Z 16 320-339 132 132 130 229 229 227 F F F <---
145842024-02-07T19:00:55.914Z 17 340-359 132 132 130 229 229 227 F F F <---
145852024-02-07T19:00:55.914Z 18 360-379 125 125 125 220 220 220 F F F
145862024-02-07T19:00:55.914Z 19 380-399 127 127 127 222 222 222 T T F <---
145872024-02-07T19:00:55.915Z 20 400-419 126 126 126 221 221 221 T T F <---
145882024-02-07T19:00:55.915Z 21 420-439 123 123 123 216 216 216 F F F
145892024-02-07T19:00:55.915Z 22 440-459 130 130 130 224 224 224 F F F
145902024-02-07T19:00:55.915Z 23 460-479 125 125 125 220 220 220 F F F
145912024-02-07T19:00:55.915Z 24 480-499 131 131 131 228 228 228 F F F
145922024-02-07T19:00:55.915Z 25 500-519 132 132 131 229 229 228 F F T <---
145932024-02-07T19:00:55.915Z 26 520-539 132 132 131 229 229 228 F F T <---
145942024-02-07T19:00:55.916Z 27 540-559 131 131 131 228 228 228 F F F
145952024-02-07T19:00:55.916Z 28 560-579 130 130 130 227 227 227 F F F
145962024-02-07T19:00:55.916Z 29 580-599 130 130 130 227 227 227 T T F <---
145972024-02-07T19:00:55.916ZMax gen: 132, Max flush: 229
145982024-02-07T19:00:55.916ZError: Difference in extent metadata found!
145992024-02-07T19:00:55.916ZA Difference in extent metadata is expected here
146002024-02-07T19:00:55.916ZOn loop 66
146012024-02-07T19:00:55.916Z
146022024-02-07T19:00:55.917ZVerifying data now
146032024-02-07T19:01:01.610ZNo change after verify
146042024-02-07T19:01:01.610ZLoop: 66 Downstairs dump after verify (and repair):
146052024-02-07T19:01:01.615ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146062024-02-07T19:01:01.615Z 0 000-019 132 132 132 229 229 229 F F F
146072024-02-07T19:01:01.616Z 1 020-039 130 130 130 224 224 224 F F F
146082024-02-07T19:01:01.616Z 2 040-059 128 128 128 223 223 223 F F F
146092024-02-07T19:01:01.616Z 3 060-079 127 127 127 222 222 222 F F F
146102024-02-07T19:01:01.616Z 4 080-099 133 133 133 230 230 230 F F F
146112024-02-07T19:01:01.616Z 5 100-119 133 133 133 230 230 230 F F F
146122024-02-07T19:01:01.616Z 6 120-139 132 132 132 229 229 229 F F F
146132024-02-07T19:01:01.616Z 7 140-159 128 128 128 223 223 223 F F F
146142024-02-07T19:01:01.616Z 8 160-179 131 131 131 228 228 228 F F F
146152024-02-07T19:01:01.616Z 9 180-199 133 133 133 230 230 230 F F F
146162024-02-07T19:01:01.625Z 10 200-219 132 132 132 229 229 229 F F F
146172024-02-07T19:01:01.626Z 11 220-239 130 130 130 227 227 227 F F F
146182024-02-07T19:01:01.626Z 12 240-259 132 132 132 229 229 229 F F F
146192024-02-07T19:01:01.626Z 13 260-279 132 132 132 229 229 229 F F F
146202024-02-07T19:01:01.626Z 14 280-299 127 127 127 222 222 222 F F F
146212024-02-07T19:01:01.626Z 15 300-319 127 127 127 222 222 222 F F F
146222024-02-07T19:01:01.626Z 16 320-339 132 132 132 229 229 229 F F F
146232024-02-07T19:01:01.626Z 17 340-359 132 132 132 229 229 229 F F F
146242024-02-07T19:01:01.627Z 18 360-379 125 125 125 220 220 220 F F F
146252024-02-07T19:01:01.627Z 19 380-399 133 133 133 230 230 230 F F F
146262024-02-07T19:01:01.627Z 20 400-419 133 133 133 230 230 230 F F F
146272024-02-07T19:01:01.627Z 21 420-439 123 123 123 216 216 216 F F F
146282024-02-07T19:01:01.627Z 22 440-459 130 130 130 224 224 224 F F F
146292024-02-07T19:01:01.627Z 23 460-479 125 125 125 220 220 220 F F F
146302024-02-07T19:01:01.627Z 24 480-499 131 131 131 228 228 228 F F F
146312024-02-07T19:01:01.627Z 25 500-519 132 132 132 229 229 229 F F F
146322024-02-07T19:01:01.627Z 26 520-539 132 132 132 229 229 229 F F F
146332024-02-07T19:01:01.627Z 27 540-559 131 131 131 228 228 228 F F F
146342024-02-07T19:01:01.627Z 28 560-579 130 130 130 227 227 227 F F F
146352024-02-07T19:01:01.627Z 29 580-599 133 133 133 230 230 230 F F F
146362024-02-07T19:01:01.627ZMax gen: 133, Max flush: 230
146372024-02-07T19:01:01.627Z
146382024-02-07T19:01:01.628ZBegin loop 67
146392024-02-07T19:01:01.628ZDownstairs to restart: 2
146402024-02-07T19:01:01.642Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.851537151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671}
146412024-02-07T19:01:01.642Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.851832181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671}
146422024-02-07T19:01:01.642Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.851854958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671}
146432024-02-07T19:01:01.654Z{"msg":"Crucible a0f0d559-1aec-4843-91b7-ea35e4f24a32 has session id: 06988674-0616-4144-98b5-0531fa5e6793","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.851878185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146442024-02-07T19:01:01.654Z{"msg":"Crucible runtime is spawned
146452024-02-07T19:01:01.654ZUpstairs opts: Upstairs UUID: a0f0d559-1aec-4843-91b7-ea35e4f24a32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.851948996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146462024-02-07T19:01:01.654Z{"msg":"Crucible stats registered with UUID: a0f0d559-1aec-4843-91b7-ea35e4f24a32","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852002818Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146472024-02-07T19:01:01.654Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852029275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671}
146482024-02-07T19:01:01.655Z{"msg":"a0f0d559-1aec-4843-91b7-ea35e4f24a32 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852100505Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146492024-02-07T19:01:01.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852129851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146502024-02-07T19:01:01.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852156198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146512024-02-07T19:01:01.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852184574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146522024-02-07T19:01:01.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85220921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146532024-02-07T19:01:01.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852242396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146542024-02-07T19:01:01.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852277421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146552024-02-07T19:01:01.655Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852302868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146562024-02-07T19:01:01.655Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852332964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146572024-02-07T19:01:01.656Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85235797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146582024-02-07T19:01:01.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852526598Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146592024-02-07T19:01:01.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852586279Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146602024-02-07T19:01:01.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.852615355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146612024-02-07T19:01:01.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.853377552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146622024-02-07T19:01:01.656Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.853423426Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146632024-02-07T19:01:01.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.853454362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146642024-02-07T19:01:01.656Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.853509945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146652024-02-07T19:01:01.656Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85413243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146662024-02-07T19:01:01.657Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854289959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146672024-02-07T19:01:01.657Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854408513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146682024-02-07T19:01:01.657Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854441019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146692024-02-07T19:01:01.657Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854687305Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146702024-02-07T19:01:01.657Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854739158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146712024-02-07T19:01:01.657Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854765925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146722024-02-07T19:01:01.657Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854791461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146732024-02-07T19:01:01.657Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854828396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146742024-02-07T19:01:01.657Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.854846844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146752024-02-07T19:01:01.657Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855110858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146762024-02-07T19:01:01.657Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85561696Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146772024-02-07T19:01:01.658Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855658274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146782024-02-07T19:01:01.658Z{"msg":"[0]R flush_numbers[0..12]: [229, 224, 223, 222, 230, 230, 229, 223, 228, 230, 229, 227]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855676601Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146792024-02-07T19:01:01.658Z{"msg":"[0]R generation[0..12]: [132, 130, 128, 127, 133, 133, 132, 128, 131, 133, 132, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855698649Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146802024-02-07T19:01:01.658Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855714067Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146812024-02-07T19:01:01.658Z{"msg":"[1]R flush_numbers[0..12]: [229, 224, 223, 222, 230, 230, 229, 223, 228, 230, 229, 227]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855737013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146822024-02-07T19:01:01.658Z{"msg":"[1]R generation[0..12]: [132, 130, 128, 127, 133, 133, 132, 128, 131, 133, 132, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855752021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146832024-02-07T19:01:01.658Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855778258Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146842024-02-07T19:01:01.658Z{"msg":"[2]R flush_numbers[0..12]: [229, 224, 223, 222, 230, 230, 229, 223, 228, 230, 229, 227]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855793236Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146852024-02-07T19:01:01.658Z{"msg":"[2]R generation[0..12]: [132, 130, 128, 127, 133, 133, 132, 128, 131, 133, 132, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855807624Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146862024-02-07T19:01:01.658Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855828821Z","Wait for a query_work_queue command to finish before sending IO
146872024-02-07T19:01:01.659Zhostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146882024-02-07T19:01:01.659Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855845789Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146892024-02-07T19:01:01.659Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855867446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146902024-02-07T19:01:01.659Z{"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855880824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146912024-02-07T19:01:01.659Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85590637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146922024-02-07T19:01:01.659Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855920259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146932024-02-07T19:01:01.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.855947725Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146942024-02-07T19:01:01.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85598012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146952024-02-07T19:01:01.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.856009127Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146962024-02-07T19:01:01.659Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.856025634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146972024-02-07T19:01:01.660Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.856038432Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146982024-02-07T19:01:01.660Z{"msg":"a0f0d559-1aec-4843-91b7-ea35e4f24a32 is now active with session: 06988674-0616-4144-98b5-0531fa5e6793","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.85605857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
146992024-02-07T19:01:01.660Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.856074958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671,"session_id":"06988674-0616-4144-98b5-0531fa5e6793"}
147002024-02-07T19:01:01.660Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:00.856098015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671}
147012024-02-07T19:01:01.660ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
147022024-02-07T19:01:01.660ZLoading write count information from file "/tmp/test_repair_verify.data"
147032024-02-07T19:01:01.660ZRun Repair workload
147042024-02-07T19:01:01.660Z01/30 Write block 575 len 28672 data: 8 7 6 7 8 9 11
147052024-02-07T19:01:01.660Z02/30 Write block 502 len 40960 data: 11 13 11 14 12 10 11 11 13 10
147062024-02-07T19:01:01.660Z03/30 Read block 169 len 28672
147072024-02-07T19:01:01.660Z04/30 Read block 291 len 28672
147082024-02-07T19:01:01.661Z05/30 Write block 245 len 12288 data: 11 12 11
147092024-02-07T19:01:01.661Z06/30 Write block 547 len 12288 data: 12 13 15
147102024-02-07T19:01:01.671Z07/30 Read block 214 len 12288
147112024-02-07T19:01:01.671Z08/30 Read block 583 len 8192
147122024-02-07T19:01:01.671Z09/30 Write block 223 len 12288 data: 9 10 8
147132024-02-07T19:01:01.671Z10/30 Read block 126 len 32768
147142024-02-07T19:01:01.671Z11/30 Write block 291 len 28672 data: 15 12 11 9 10 9 11
147152024-02-07T19:01:01.671Z12/30 Read block 533 len 28672
147162024-02-07T19:01:01.671Z13/30 Write block 23 len 8192 data: 16 13
147172024-02-07T19:01:01.671Z14/30 Read block 402 len 32768
147182024-02-07T19:01:01.672Z15/30 Read block 324 len 4096
147192024-02-07T19:01:01.672Z16/30 Read block 531 len 36864
147202024-02-07T19:01:01.672Z17/30 Read block 265 len 32768
147212024-02-07T19:01:01.672Z18/30 Read block 367 len 24576
147222024-02-07T19:01:01.672Z19/30 Read block 536 len 20480
147232024-02-07T19:01:01.672Z20/30 Read block 93 len 4096
147242024-02-07T19:01:01.672Z21/30 Read block 578 len 12288
147252024-02-07T19:01:01.672Z22/30 Read block 260 len 32768
147262024-02-07T19:01:01.672Z23/30 Write block 501 len 24576 data: 8 12 14 12 15 13
147272024-02-07T19:01:01.672Z24/30 Write block 430 len 16384 data: 10 10 7 5
147282024-02-07T19:01:01.672Z25/30 Read block 68 len 36864
147292024-02-07T19:01:01.672Z26/30 Write block 188 len 16384 data: 9 9 9 13
147302024-02-07T19:01:01.673Z27/30 Write block 405 len 20480 data: 9 10 9 9 10
147312024-02-07T19:01:01.673Z28/30 Read block 199 len 36864
147322024-02-07T19:01:01.673Z29/30 Write block 43 len 4096 data: 7
147332024-02-07T19:01:01.673Z30/30 Read block 109 len 12288
147342024-02-07T19:01:01.673Z----------------------------------------------------------------
147352024-02-07T19:01:01.673Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
147362024-02-07T19:01:01.673ZStates: Active Active Active
147372024-02-07T19:01:01.673ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147382024-02-07T19:01:01.673Z 1 Acked 1000 Write 7 Done Done Done false
147392024-02-07T19:01:01.673Z 2 Acked 1001 Write 10 Done Done Done false
147402024-02-07T19:01:01.673Z 3 Acked 1002 Read 7 Done Done Done false
147412024-02-07T19:01:01.673Z 4 Acked 1003 Read 7 Done Done Sent false
147422024-02-07T19:01:01.673Z 5 Acked 1004 Write 3 Done Done Sent false
147432024-02-07T19:01:01.674Z 6 Acked 1005 Write 3 Done Done Sent false
147442024-02-07T19:01:01.674Z 7 Acked 1006 Read 3 Done Done Sent false
147452024-02-07T19:01:01.674Z 8 Acked 1007 Read 2 Done Done Sent false
147462024-02-07T19:01:01.674Z 9 Acked 1008 Write 3 Done Done Sent false
147472024-02-07T19:01:01.674Z 10 Acked 1009 Read 8 Done Done Sent false
147482024-02-07T19:01:01.674Z 11 Acked 1010 Write 7 Done Done Sent false
147492024-02-07T19:01:01.674Z 12 Acked 1011 Read 7 Done Done Sent false
147502024-02-07T19:01:01.674Z 13 Acked 1012 Write 2 Done Done Sent false
147512024-02-07T19:01:01.674Z 14 Acked 1013 Read 8 Done Done Sent false
147522024-02-07T19:01:01.674Z 15 Acked 1014 Read 1 Done Done Sent false
147532024-02-07T19:01:01.674Z 16 Acked 1015 Read 9 Done Done Sent false
147542024-02-07T19:01:01.674Z 17 Acked 1016 Read 8 Done Done Sent false
147552024-02-07T19:01:01.674Z 18 Acked 1017 Read 6 Done Done Sent false
147562024-02-07T19:01:01.675Z 19 Acked 1018 Read 5 Done Done Sent false
147572024-02-07T19:01:01.675Z 20 Acked 1019 Read 1 Done Done Sent false
147582024-02-07T19:01:01.675Z 21 Acked 1020 Read 3 Done Done Sent false
147592024-02-07T19:01:01.675Z 22 Acked 1021 Read 8 Done Done Sent false
147602024-02-07T19:01:01.675Z 23 Acked 1022 Write 6 Done Done Sent false
147612024-02-07T19:01:01.675Z 24 Acked 1023 Write 4 Done Done Sent false
147622024-02-07T19:01:01.675Z 25 Acked 1024 Read 9 Done Done Sent false
147632024-02-07T19:01:01.675Z 26 Acked 1025 Write 4 Done Done Sent false
147642024-02-07T19:01:01.675Z 27 Acked 1026 Write 5 Done Done Sent false
147652024-02-07T19:01:01.675Z 28 Acked 1027 Read 9 Done Done Sent false
147662024-02-07T19:01:01.675Z 29 Acked 1028 Write 1 Done Done Sent false
147672024-02-07T19:01:01.675Z 30 Acked 1029 Read 3 Done Sent Sent false
147682024-02-07T19:01:01.675Z STATES DS:0 DS:1 DS:2 TOTAL
147692024-02-07T19:01:01.676Z New 0 0 0 0
147702024-02-07T19:01:01.676Z Sent 0 1 27 28
147712024-02-07T19:01:01.676Z Done 30 29 3 62
147722024-02-07T19:01:01.676Z Skipped 0 0 0 0
147732024-02-07T19:01:01.676Z Error 0 0 0 0
147742024-02-07T19:01:01.676ZLast Flush: 0 0 0
147752024-02-07T19:01:01.676ZDownstairs last five completed:
147762024-02-07T19:01:01.676ZUpstairs last five completed: 30 29 28 27 26
147772024-02-07T19:01:01.676Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:00.882801183Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1671}
147782024-02-07T19:01:01.676ZWrote out file "/tmp/test_repair_verify.data"
147792024-02-07T19:01:01.681Z
147802024-02-07T19:01:01.681ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147812024-02-07T19:01:01.718ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147822024-02-07T19:01:01.718Z 0 000-019 132 132 132 229 229 229 F F F
147832024-02-07T19:01:01.719Z 1 020-039 130 130 130 224 224 224 T T F <---
147842024-02-07T19:01:01.719Z 2 040-059 128 128 128 223 223 223 T T F <---
147852024-02-07T19:01:01.719Z 3 060-079 127 127 127 222 222 222 F F F
147862024-02-07T19:01:01.719Z 4 080-099 133 133 133 230 230 230 F F F
147872024-02-07T19:01:01.719Z 5 100-119 133 133 133 230 230 230 F F F
147882024-02-07T19:01:01.719Z 6 120-139 132 132 132 229 229 229 F F F
147892024-02-07T19:01:01.719Z 7 140-159 128 128 128 223 223 223 F F F
147902024-02-07T19:01:01.719Z 8 160-179 131 131 131 228 228 228 F F F
147912024-02-07T19:01:01.719Z 9 180-199 133 133 133 230 230 230 T T F <---
147922024-02-07T19:01:01.720Z 10 200-219 132 132 132 229 229 229 F F F
147932024-02-07T19:01:01.720Z 11 220-239 130 130 130 227 227 227 T T F <---
147942024-02-07T19:01:01.720Z 12 240-259 132 132 132 229 229 229 T T F <---
147952024-02-07T19:01:01.720Z 13 260-279 132 132 132 229 229 229 F F F
147962024-02-07T19:01:01.720Z 14 280-299 127 127 127 222 222 222 T T F <---
147972024-02-07T19:01:01.720Z 15 300-319 127 127 127 222 222 222 F F F
147982024-02-07T19:01:01.720Z 16 320-339 132 132 132 229 229 229 F F F
147992024-02-07T19:01:01.720Z 17 340-359 132 132 132 229 229 229 F F F
148002024-02-07T19:01:01.720Z 18 360-379 125 125 125 220 220 220 F F F
148012024-02-07T19:01:01.720Z 19 380-399 133 133 133 230 230 230 F F F
148022024-02-07T19:01:01.721Z 20 400-419 133 133 133 230 230 230 T T F <---
148032024-02-07T19:01:01.721Z 21 420-439 123 123 123 216 216 216 T T F <---
148042024-02-07T19:01:01.721Z 22 440-459 130 130 130 224 224 224 F F F
148052024-02-07T19:01:01.721Z 23 460-479 125 125 125 220 220 220 F F F
148062024-02-07T19:01:01.721Z 24 480-499 131 131 131 228 228 228 F F F
148072024-02-07T19:01:01.721Z 25 500-519 132 132 132 229 229 229 T T T
148082024-02-07T19:01:01.721Z 26 520-539 132 132 132 229 229 229 F F F
148092024-02-07T19:01:01.721Z 27 540-559 131 131 131 228 228 228 T T F <---
148102024-02-07T19:01:01.721Z 28 560-579 130 130 130 227 227 227 T T T
148112024-02-07T19:01:01.721Z 29 580-599 133 133 133 230 230 230 T T T
148122024-02-07T19:01:01.722ZMax gen: 133, Max flush: 230
148132024-02-07T19:01:01.722ZError: Difference in extent metadata found!
148142024-02-07T19:01:01.722ZA Difference in extent metadata is expected here
148152024-02-07T19:01:01.722ZOn loop 67
148162024-02-07T19:01:01.722Z
148172024-02-07T19:01:01.722ZVerifying data now
148182024-02-07T19:01:07.311ZNo change after verify
148192024-02-07T19:01:07.311ZLoop: 67 Downstairs dump after verify (and repair):
148202024-02-07T19:01:07.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148212024-02-07T19:01:07.323Z 0 000-019 132 132 132 229 229 229 F F F
148222024-02-07T19:01:07.323Z 1 020-039 134 134 134 231 231 231 F F F
148232024-02-07T19:01:07.323Z 2 040-059 134 134 134 231 231 231 F F F
148242024-02-07T19:01:07.323Z 3 060-079 127 127 127 222 222 222 F F F
148252024-02-07T19:01:07.323Z 4 080-099 133 133 133 230 230 230 F F F
148262024-02-07T19:01:07.323Z 5 100-119 133 133 133 230 230 230 F F F
148272024-02-07T19:01:07.324Z 6 120-139 132 132 132 229 229 229 F F F
148282024-02-07T19:01:07.324Z 7 140-159 128 128 128 223 223 223 F F F
148292024-02-07T19:01:07.324Z 8 160-179 131 131 131 228 228 228 F F F
148302024-02-07T19:01:07.324Z 9 180-199 134 134 134 231 231 231 F F F
148312024-02-07T19:01:07.324Z 10 200-219 132 132 132 229 229 229 F F F
148322024-02-07T19:01:07.332Z 11 220-239 134 134 134 231 231 231 F F F
148332024-02-07T19:01:07.332Z 12 240-259 134 134 134 231 231 231 F F F
148342024-02-07T19:01:07.332Z 13 260-279 132 132 132 229 229 229 F F F
148352024-02-07T19:01:07.333Z 14 280-299 134 134 134 231 231 231 F F F
148362024-02-07T19:01:07.333Z 15 300-319 127 127 127 222 222 222 F F F
148372024-02-07T19:01:07.333Z 16 320-339 132 132 132 229 229 229 F F F
148382024-02-07T19:01:07.333Z 17 340-359 132 132 132 229 229 229 F F F
148392024-02-07T19:01:07.333Z 18 360-379 125 125 125 220 220 220 F F F
148402024-02-07T19:01:07.333Z 19 380-399 133 133 133 230 230 230 F F F
148412024-02-07T19:01:07.333Z 20 400-419 134 134 134 231 231 231 F F F
148422024-02-07T19:01:07.334Z 21 420-439 134 134 134 231 231 231 F F F
148432024-02-07T19:01:07.334Z 22 440-459 130 130 130 224 224 224 F F F
148442024-02-07T19:01:07.334Z 23 460-479 125 125 125 220 220 220 F F F
148452024-02-07T19:01:07.334Z 24 480-499 131 131 131 228 228 228 F F F
148462024-02-07T19:01:07.334Z 25 500-519 134 134 134 231 231 231 F F F
148472024-02-07T19:01:07.334Z 26 520-539 132 132 132 229 229 229 F F F
148482024-02-07T19:01:07.334Z 27 540-559 134 134 134 231 231 231 F F F
148492024-02-07T19:01:07.335Z 28 560-579 134 134 134 231 231 231 F F F
148502024-02-07T19:01:07.335Z 29 580-599 134 134 134 231 231 231 F F F
148512024-02-07T19:01:07.335ZMax gen: 134, Max flush: 231
148522024-02-07T19:01:07.335Z
148532024-02-07T19:01:07.335ZBegin loop 68
148542024-02-07T19:01:07.335ZDownstairs to restart: 0
148552024-02-07T19:01:07.343Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558061511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679}
148562024-02-07T19:01:07.343Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
148572024-02-07T19:01:07.343Z2024-02-07T19:01:06.558430141Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679}
148582024-02-07T19:01:07.352Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558477335Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679}
148592024-02-07T19:01:07.352Z{"msg":"Crucible 8ab0f6f4-c23b-4db6-8bb8-c61b4fd1cf8b has session id: 6392c042-76ad-47b2-a711-7a5d7c0cacf1","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.55851312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148602024-02-07T19:01:07.352Z{"msg":"Upstairs opts: Upstairs UUID: 8ab0f6f4-c23b-4db6-8bb8-c61b4fd1cf8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558572852Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148612024-02-07T19:01:07.352Z{"msg":"Crucible stats registered with UUID: 8ab0f6f4-c23b-4db6-8bb8-c61b4fd1cf8b","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558622775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148622024-02-07T19:01:07.352Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558645702Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679}
148632024-02-07T19:01:07.352Z{"msg":"8ab0f6f4-c23b-4db6-8bb8-c61b4fd1cf8b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558678508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148642024-02-07T19:01:07.352Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558703724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148652024-02-07T19:01:07.353Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558731671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148662024-02-07T19:01:07.353Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558761796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148672024-02-07T19:01:07.353Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558802641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148682024-02-07T19:01:07.353Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558822868Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148692024-02-07T19:01:07.353Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558853114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148702024-02-07T19:01:07.353Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.55888214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148712024-02-07T19:01:07.353Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558913846Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148722024-02-07T19:01:07.354Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.558990225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148732024-02-07T19:01:07.354Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559039119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148742024-02-07T19:01:07.354Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559087972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148752024-02-07T19:01:07.354Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559118088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148762024-02-07T19:01:07.354Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559743014Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148772024-02-07T19:01:07.354Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559785898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148782024-02-07T19:01:07.354Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559816614Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148792024-02-07T19:01:07.354Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.559854908Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148802024-02-07T19:01:07.355Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560309977Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148812024-02-07T19:01:07.355Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560453938Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148822024-02-07T19:01:07.355Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560802171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148832024-02-07T19:01:07.355Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560855013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148842024-02-07T19:01:07.355Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560919415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148852024-02-07T19:01:07.355Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560938932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148862024-02-07T19:01:07.355Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560961249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148872024-02-07T19:01:07.355Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.560976387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148882024-02-07T19:01:07.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.561004093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148892024-02-07T19:01:07.355Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.561020921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148902024-02-07T19:01:07.356Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.561495377Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148912024-02-07T19:01:07.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.561953025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148922024-02-07T19:01:07.356Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.561998539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148932024-02-07T19:01:07.356Z{"msg":"[0]R flush_numbers[0..12]: [229, 231, 231, 222, 230, 230, 229, 223, 228, 231, 229, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562026485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148942024-02-07T19:01:07.356Z{"msg":"[0]R generation[0..12]: [132, 134, 134, 127, 133, 133, 132, 128, 131, 134, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.5620634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148952024-02-07T19:01:07.356Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562100445Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148962024-02-07T19:01:07.356Z{"msg":"[1]R flush_numbers[0..12]: [229, 231, 231, 222, 230, 230, 229, 223, 228, 231, 229, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.56213842Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148972024-02-07T19:01:07.356Z{"msg":"[1]R generation[0..12]: [132, 134, 134, 127, 133, 133, 132, 128, 131, 134, 132, 134]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
148982024-02-07T19:01:07.357Z,"time":"2024-02-07T19:01:06.562185933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
148992024-02-07T19:01:07.357Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562232207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149002024-02-07T19:01:07.357Z{"msg":"[2]R flush_numbers[0..12]: [229, 231, 231, 222, 230, 230, 229, 223, 228, 231, 229, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562271502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149012024-02-07T19:01:07.357Z{"msg":"[2]R generation[0..12]: [132, 134, 134, 127, 133, 133, 132, 128, 131, 134, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562311726Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149022024-02-07T19:01:07.357Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562350431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149032024-02-07T19:01:07.357Z{"msg":"Max found gen is 135","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562387906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149042024-02-07T19:01:07.357Z{"msg":"Generation requested: 136 >= found:135","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562425881Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149052024-02-07T19:01:07.357Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562464216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149062024-02-07T19:01:07.358Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562500121Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149072024-02-07T19:01:07.358Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562537156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149082024-02-07T19:01:07.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562572871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149092024-02-07T19:01:07.358ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
149102024-02-07T19:01:07.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562613606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149112024-02-07T19:01:07.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562640722Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149122024-02-07T19:01:07.358Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.56265525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149132024-02-07T19:01:07.358Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562674797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149142024-02-07T19:01:07.358Z{"msg":"8ab0f6f4-c23b-4db6-8bb8-c61b4fd1cf8b is now active with session: 6392c042-76ad-47b2-a711-7a5d7c0cacf1","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562688375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149152024-02-07T19:01:07.358Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562707193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679,"session_id":"6392c042-76ad-47b2-a711-7a5d7c0cacf1"}
149162024-02-07T19:01:07.359Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:06.562720381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679}
149172024-02-07T19:01:07.359ZLoading write count information from file "/tmp/test_repair_verify.data"
149182024-02-07T19:01:07.359ZRun Repair workload
149192024-02-07T19:01:07.365Z01/30 Write block 318 len 24576 data: 12 10 9 8 9 8
149202024-02-07T19:01:07.365Z02/30 Write block 514 len 32768 data: 14 13 12 9 11 10 10 7
149212024-02-07T19:01:07.366Z03/30 Write block 462 len 12288 data: 7 8 8
149222024-02-07T19:01:07.366Z04/30 Write block 431 len 16384 data: 11 8 6 4
149232024-02-07T19:01:07.366Z05/30 Write block 418 len 12288 data: 10 7 7
149242024-02-07T19:01:07.366Z06/30 Write block 96 len 24576 data: 14 12 11 10 8 7
149252024-02-07T19:01:07.366Z07/30 Write block 223 len 32768 data: 10 11 9 6 6 7 9 8
149262024-02-07T19:01:07.366Z08/30 Flush
149272024-02-07T19:01:07.366Z09/30 Write block 362 len 36864 data: 12 11 8 7 10 9 10 13 16
149282024-02-07T19:01:07.366Z10/30 Write block 398 len 12288 data: 11 14 15
149292024-02-07T19:01:07.367Z11/30 Read block 58 len 24576
149302024-02-07T19:01:07.367Z12/30 Read block 306 len 8192
149312024-02-07T19:01:07.367Z13/30 Flush
149322024-02-07T19:01:07.367Z14/30 Flush
149332024-02-07T19:01:07.367Z15/30 Write block 200 len 24576 data: 9 9 11 9 11 11
149342024-02-07T19:01:07.367Z16/30 Write block 331 len 8192 data: 11 10
149352024-02-07T19:01:07.367Z17/30 Read block 166 len 24576
149362024-02-07T19:01:07.374Z18/30 Read block 496 len 8192
149372024-02-07T19:01:07.374Z19/30 Write block 577 len 36864 data: 7 8 9 10 12 12 10 11 12
149382024-02-07T19:01:07.374Z20/30 Write block 488 len 8192 data: 10 8
149392024-02-07T19:01:07.374Z21/30 Read block 74 len 8192
149402024-02-07T19:01:07.374Z22/30 Read block 379 len 16384
149412024-02-07T19:01:07.374Z23/30 Read block 547 len 28672
149422024-02-07T19:01:07.374Z24/30 Read block 569 len 24576
149432024-02-07T19:01:07.374Z25/30 Write block 247 len 36864 data: 12 11 10 9 8 9 9 10 9
149442024-02-07T19:01:07.375Z26/30 Write block 62 len 20480 data: 6 7 10 10 13
149452024-02-07T19:01:07.375Z27/30 Read block 217 len 4096
149462024-02-07T19:01:07.381Z28/30 Read block 112 len 40960
149472024-02-07T19:01:07.381Z29/30 Write block 115 len 24576 data: 9 11 13 14 16 14
149482024-02-07T19:01:07.381Z30/30 Write block 432 len 32768 data: 9 7 5 6 9 11 7 10
149492024-02-07T19:01:07.381Z----------------------------------------------------------------
149502024-02-07T19:01:07.381Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
149512024-02-07T19:01:07.381ZStates: Active Active Active
149522024-02-07T19:01:07.381ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149532024-02-07T19:01:07.381Z 1 Acked 1000 Write 6 Done Done Done false
149542024-02-07T19:01:07.381Z 2 Acked 1001 Write 8 Sent Done Done false
149552024-02-07T19:01:07.381Z 3 Acked 1002 Write 3 Sent Done Done false
149562024-02-07T19:01:07.381Z 4 Acked 1003 Write 4 Sent Done Done false
149572024-02-07T19:01:07.382Z 5 Acked 1004 Write 3 Sent Done Done false
149582024-02-07T19:01:07.382Z 6 Acked 1005 Write 6 Sent Done Done false
149592024-02-07T19:01:07.382Z 7 Acked 1006 Write 8 Sent Done Done false
149602024-02-07T19:01:07.382Z 8 Acked 1007 Flush 0 Sent Done Done false
149612024-02-07T19:01:07.382Z 9 Acked 1008 Write 9 Sent Done Done false
149622024-02-07T19:01:07.382Z 10 Acked 1009 Write 3 Sent Done Done false
149632024-02-07T19:01:07.382Z 11 Acked 1010 Read 6 Sent Done Done false
149642024-02-07T19:01:07.382Z 12 Acked 1011 Read 2 Sent Done Done false
149652024-02-07T19:01:07.382Z 13 Acked 1012 Flush 0 Sent Done Done false
149662024-02-07T19:01:07.382Z 14 Acked 1013 Flush 0 Sent Done Done false
149672024-02-07T19:01:07.382Z 15 Acked 1014 Write 6 Sent Done Done false
149682024-02-07T19:01:07.382Z 16 Acked 1015 Write 2 Sent Done Done false
149692024-02-07T19:01:07.383Z 17 Acked 1016 Read 6 Sent Done Done false
149702024-02-07T19:01:07.383Z 18 Acked 1017 Read 2 Sent Done Done false
149712024-02-07T19:01:07.383Z 19 Acked 1018 Write 9 Sent Done Done false
149722024-02-07T19:01:07.383Z 20 Acked 1019 Write 2 Sent Done Done false
149732024-02-07T19:01:07.383Z 21 Acked 1020 Read 2 Sent Done Done false
149742024-02-07T19:01:07.383Z 22 Acked 1021 Read 4 Sent Done Done false
149752024-02-07T19:01:07.383Z 23 Acked 1022 Read 7 Sent Done Done false
149762024-02-07T19:01:07.383Z 24 Acked 1023 Read 6 Sent Done Done false
149772024-02-07T19:01:07.383Z 25 Acked 1024 Write 9 Sent Done Done false
149782024-02-07T19:01:07.383Z 26 Acked 1025 Write 5 Sent Done Done false
149792024-02-07T19:01:07.383Z 27 Acked 1026 Read 1 Sent Done Done false
149802024-02-07T19:01:07.383Z 28 Acked 1027 Read 10 Sent Done Done false
149812024-02-07T19:01:07.384Z 29 Acked 1028 Write 6 Sent Done Done false
149822024-02-07T19:01:07.384Z 30 Acked 1029 Write 8 Sent Sent Sent false
149832024-02-07T19:01:07.384Z STATES DS:0 DS:1 DS:2 TOTAL
149842024-02-07T19:01:07.384Z New 0 0 0 0
149852024-02-07T19:01:07.384Z Sent 29 1 1 31
149862024-02-07T19:01:07.384Z Done 1 29 29 59
149872024-02-07T19:01:07.384Z Skipped 0 0 0 0
149882024-02-07T19:01:07.384Z Error 0 0 0 0
149892024-02-07T19:01:07.384ZLast Flush: 0 1013 1013
149902024-02-07T19:01:07.384ZDownstairs last five completed:
149912024-02-07T19:01:07.384ZUpstairs last five completed: 30 29 28 27 26
149922024-02-07T19:01:07.385Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:06.592441071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1679}
149932024-02-07T19:01:07.385ZWrote out file "/tmp/test_repair_verify.data"
149942024-02-07T19:01:07.391Z
149952024-02-07T19:01:07.392ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149962024-02-07T19:01:07.408ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149972024-02-07T19:01:07.408Z 0 000-019 132 132 132 229 229 229 F F F
149982024-02-07T19:01:07.409Z 1 020-039 134 134 134 231 231 231 F F F
149992024-02-07T19:01:07.409Z 2 040-059 134 134 134 231 231 231 F F F
150002024-02-07T19:01:07.409Z 3 060-079 127 127 127 222 222 222 F T T <---
150012024-02-07T19:01:07.409Z 4 080-099 133 136 136 230 232 232 F F F <---
150022024-02-07T19:01:07.410Z 5 100-119 133 136 136 230 232 232 F T T <---
150032024-02-07T19:01:07.410Z 6 120-139 132 132 132 229 229 229 F T T <---
150042024-02-07T19:01:07.410Z 7 140-159 128 128 128 223 223 223 F F F
150052024-02-07T19:01:07.410Z 8 160-179 131 131 131 228 228 228 F F F
150062024-02-07T19:01:07.410Z 9 180-199 134 134 134 231 231 231 F F F
150072024-02-07T19:01:07.411Z 10 200-219 132 132 132 229 229 229 F T T <---
150082024-02-07T19:01:07.411Z 11 220-239 134 136 136 231 232 232 F F F <---
150092024-02-07T19:01:08.250Z 12 240-259 134 134 134 231 231 231 F T T <---
150102024-02-07T19:01:08.251Z 13 260-279 132 132 132 229 229 229 F F F
150112024-02-07T19:01:08.251Z 14 280-299 134 134 134 231 231 231 F F F
150122024-02-07T19:01:08.251Z 15 300-319 127 136 136 222 232 232 T F F <---
150132024-02-07T19:01:08.251Z 16 320-339 132 136 136 229 232 232 T T T <---
150142024-02-07T19:01:08.251Z 17 340-359 132 132 132 229 229 229 F F F
150152024-02-07T19:01:08.251Z 18 360-379 125 136 136 220 233 233 F F F <---
150162024-02-07T19:01:08.251Z 19 380-399 133 136 136 230 233 233 F F F <---
150172024-02-07T19:01:08.251Z 20 400-419 134 136 136 231 233 233 F F F <---
150182024-02-07T19:01:08.251Z 21 420-439 134 136 136 231 232 232 F T T <---
150192024-02-07T19:01:08.252Z 22 440-459 130 130 130 224 224 224 F F F
150202024-02-07T19:01:08.252Z 23 460-479 125 136 136 220 232 232 F F F <---
150212024-02-07T19:01:08.252Z 24 480-499 131 131 131 228 228 228 F T T <---
150222024-02-07T19:01:08.252Z 25 500-519 134 136 136 231 232 232 F F F <---
150232024-02-07T19:01:08.252Z 26 520-539 132 136 136 229 232 232 F F F <---
150242024-02-07T19:01:08.252Z 27 540-559 134 134 134 231 231 231 F F F
150252024-02-07T19:01:08.252Z 28 560-579 134 134 134 231 231 231 F T T <---
150262024-02-07T19:01:08.252Z 29 580-599 134 134 134 231 231 231 F T T <---
150272024-02-07T19:01:08.252ZMax gen: 136, Max flush: 233
150282024-02-07T19:01:08.252ZError: Difference in extent metadata found!
150292024-02-07T19:01:08.253ZA Difference in extent metadata is expected here
150302024-02-07T19:01:08.253ZOn loop 68
150312024-02-07T19:01:08.253Z
150322024-02-07T19:01:08.253ZVerifying data now
150332024-02-07T19:01:13.285ZNo change after verify
150342024-02-07T19:01:13.285ZLoop: 68 Downstairs dump after verify (and repair):
150352024-02-07T19:01:13.300ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150362024-02-07T19:01:13.300Z 0 000-019 132 132 132 229 229 229 F F F
150372024-02-07T19:01:13.300Z 1 020-039 134 134 134 231 231 231 F F F
150382024-02-07T19:01:13.300Z 2 040-059 134 134 134 231 231 231 F F F
150392024-02-07T19:01:13.300Z 3 060-079 137 137 137 234 234 234 F F F
150402024-02-07T19:01:13.300Z 4 080-099 136 136 136 232 232 232 F F F
150412024-02-07T19:01:13.300Z 5 100-119 137 137 137 234 234 234 F F F
150422024-02-07T19:01:13.301Z 6 120-139 137 137 137 234 234 234 F F F
150432024-02-07T19:01:13.301Z 7 140-159 128 128 128 223 223 223 F F F
150442024-02-07T19:01:13.301Z 8 160-179 131 131 131 228 228 228 F F F
150452024-02-07T19:01:13.301Z 9 180-199 134 134 134 231 231 231 F F F
150462024-02-07T19:01:13.301Z 10 200-219 137 137 137 234 234 234 F F F
150472024-02-07T19:01:13.301Z 11 220-239 136 136 136 232 232 232 F F F
150482024-02-07T19:01:13.301Z 12 240-259 137 137 137 234 234 234 F F F
150492024-02-07T19:01:13.306Z 13 260-279 132 132 132 229 229 229 F F F
150502024-02-07T19:01:13.307Z 14 280-299 134 134 134 231 231 231 F F F
150512024-02-07T19:01:13.307Z 15 300-319 136 136 136 232 232 232 F F F
150522024-02-07T19:01:13.307Z 16 320-339 137 137 137 234 234 234 F F F
150532024-02-07T19:01:13.307Z 17 340-359 132 132 132 229 229 229 F F F
150542024-02-07T19:01:13.307Z 18 360-379 136 136 136 233 233 233 F F F
150552024-02-07T19:01:13.307Z 19 380-399 136 136 136 233 233 233 F F F
150562024-02-07T19:01:13.307Z 20 400-419 136 136 136 233 233 233 F F F
150572024-02-07T19:01:13.307Z 21 420-439 137 137 137 234 234 234 F F F
150582024-02-07T19:01:13.307Z 22 440-459 130 130 130 224 224 224 F F F
150592024-02-07T19:01:13.307Z 23 460-479 136 136 136 232 232 232 F F F
150602024-02-07T19:01:13.307Z 24 480-499 137 137 137 234 234 234 F F F
150612024-02-07T19:01:13.307Z 25 500-519 136 136 136 232 232 232 F F F
150622024-02-07T19:01:13.307Z 26 520-539 136 136 136 232 232 232 F F F
150632024-02-07T19:01:13.307Z 27 540-559 134 134 134 231 231 231 F F F
150642024-02-07T19:01:13.307Z 28 560-579 137 137 137 234 234 234 F F F
150652024-02-07T19:01:13.307Z 29 580-599 137 137 137 234 234 234 F F F
150662024-02-07T19:01:13.307ZMax gen: 137, Max flush: 234
150672024-02-07T19:01:13.308Z
150682024-02-07T19:01:13.308ZBegin loop 69
150692024-02-07T19:01:13.308ZDownstairs to restart: 1
150702024-02-07T19:01:13.320Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534298871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687}
150712024-02-07T19:01:13.320Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534562055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687}
150722024-02-07T19:01:13.320Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534589452Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687}
150732024-02-07T19:01:13.320ZCrucible runtime is spawned
150742024-02-07T19:01:13.328Z{"msg":"Crucible d41ee6fd-7984-4099-af80-47a4a2cb3ac6 has session id: aa7792ed-e2fd-444b-9412-b0e155083c57","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534639485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150752024-02-07T19:01:13.328Z{"msg":"Upstairs opts: Upstairs UUID: d41ee6fd-7984-4099-af80-47a4a2cb3ac6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534686668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150762024-02-07T19:01:13.328Z{"msg":"Crucible stats registered with UUID: d41ee6fd-7984-4099-af80-47a4a2cb3ac6","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534736422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150772024-02-07T19:01:13.328Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534767777Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687}
150782024-02-07T19:01:13.328Z{"msg":"d41ee6fd-7984-4099-af80-47a4a2cb3ac6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534817461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150792024-02-07T19:01:13.328Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534852606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150802024-02-07T19:01:13.328Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534872434Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150812024-02-07T19:01:13.328Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534907799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150822024-02-07T19:01:13.328Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534941604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150832024-02-07T19:01:13.328Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.534997146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150842024-02-07T19:01:13.328Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535030032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150852024-02-07T19:01:13.328Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535059968Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150862024-02-07T19:01:13.328Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535090694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150872024-02-07T19:01:13.329Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.53511561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150882024-02-07T19:01:13.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535143357Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150892024-02-07T19:01:13.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535176872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150902024-02-07T19:01:13.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535210888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150912024-02-07T19:01:13.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535938859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150922024-02-07T19:01:13.329Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.535987992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150932024-02-07T19:01:13.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.536020458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150942024-02-07T19:01:13.329Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.536061203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150952024-02-07T19:01:13.329Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.536586222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150962024-02-07T19:01:13.329Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.536751429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150972024-02-07T19:01:13.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537030032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150982024-02-07T19:01:13.329Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537072106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
150992024-02-07T19:01:13.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537194519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151002024-02-07T19:01:13.330Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537234804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151012024-02-07T19:01:13.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.53726154Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151022024-02-07T19:01:13.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537293056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151032024-02-07T19:01:13.330Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537318133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151042024-02-07T19:01:13.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537346309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151052024-02-07T19:01:13.330Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.537590906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151062024-02-07T19:01:13.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538032886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151072024-02-07T19:01:13.330Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.53807431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151082024-02-07T19:01:13.330Z{"msg":"[0]R flush_numbers[0..12]: [229, 231, 231, 234, 232, 234, 234, 223, 228, 231, 234, 232]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538092808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151092024-02-07T19:01:13.331Z{"msg":"[0]R generation[0..12]: [132, 134, 134, 137, 136, 137, 137, 128, 131, 134, 137, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538115205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151102024-02-07T19:01:13.331Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538130383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151112024-02-07T19:01:13.331Z{"msg":"[1]R flush_numbers[0..12]: [229, 231, 231, 234, 232, 234, 234, 223, 228, 231, 234, 232]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.53815155Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151122024-02-07T19:01:13.331Z{"msg":"[1]R generation[0..12]: [132, 134, 134, 137, 136, 137, 137, 128, 131, 134, 137, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538166308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151132024-02-07T19:01:13.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538187835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151142024-02-07T19:01:13.331Z{"msg":"[2]R flush_numbers[0..12]: [229, 231, 231, 234, 232, 234, 234, 223, 228, 231, 234, 232]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
151152024-02-07T19:01:13.331Z,"time":"2024-02-07T19:01:12.53822366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151162024-02-07T19:01:13.331Z{"msg":"[2]R generation[0..12]: [132, 134, 134, 137, 136, 137, 137, 128, 131, 134, 137, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538261835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151172024-02-07T19:01:13.331Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538290851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151182024-02-07T19:01:13.331Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538305319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151192024-02-07T19:01:13.331Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538325846Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151202024-02-07T19:01:13.332Z{"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538345884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151212024-02-07T19:01:13.332Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538382189Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151222024-02-07T19:01:13.332Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538413095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151232024-02-07T19:01:13.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538427873Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151242024-02-07T19:01:13.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538462318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151252024-02-07T19:01:13.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538497583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151262024-02-07T19:01:13.332Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.53852241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151272024-02-07T19:01:13.332Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538535558Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151282024-02-07T19:01:13.332Z{"msg":"d41ee6fd-7984-4099-af80-47a4a2cb3ac6 is now active with session: aa7792ed-e2fd-444b-9412-b0e155083c57","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538564824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151292024-02-07T19:01:13.332Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.53859823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687,"session_id":"aa7792ed-e2fd-444b-9412-b0e155083c57"}
151302024-02-07T19:01:13.332Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:12.538614727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687}
151312024-02-07T19:01:13.332ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
151322024-02-07T19:01:13.342ZLoading write count information from file "/tmp/test_repair_verify.data"
151332024-02-07T19:01:13.343ZRun Repair workload
151342024-02-07T19:01:13.343Z01/30 Write block 232 len 40960 data: 13 13 13 20 19 20 18 18 19 20
151352024-02-07T19:01:13.343Z02/30 Write block 76 len 40960 data: 7 9 11 10 8 6 6 6 5 4
151362024-02-07T19:01:13.343Z03/30 Read block 534 len 36864
151372024-02-07T19:01:13.343Z04/30 Read block 179 len 40960
151382024-02-07T19:01:13.343Z05/30 Write block 539 len 4096 data: 12
151392024-02-07T19:01:13.343Z06/30 Write block 78 len 32768 data: 12 11 9 7 7 7 6 5
151402024-02-07T19:01:13.343Z07/30 Write block 155 len 40960 data: 9 9 7 9 8 8 9 10 12 15
151412024-02-07T19:01:13.344Z08/30 Read block 201 len 16384
151422024-02-07T19:01:13.344Z09/30 Write block 487 len 32768 data: 9 11 9 9 9 9 9 12
151432024-02-07T19:01:13.344Z10/30 Read block 411 len 8192
151442024-02-07T19:01:13.344Z11/30 Write block 367 len 8192 data: 10 11
151452024-02-07T19:01:13.344Z12/30 Read block 262 len 28672
151462024-02-07T19:01:13.344Z13/30 Read block 217 len 16384
151472024-02-07T19:01:13.348Z14/30 Write block 341 len 40960 data: 10 10 10 10 13 15 15 14 13 14
151482024-02-07T19:01:13.348Z15/30 Read block 221 len 8192
151492024-02-07T19:01:13.348Z16/30 Read block 416 len 40960
151502024-02-07T19:01:13.348Z17/30 Write block 199 len 28672 data: 11 10 10 12 10 12 12
151512024-02-07T19:01:13.349Z18/30 Flush
151522024-02-07T19:01:13.349Z19/30 Write block 25 len 36864 data: 12 10 10 12 16 15 16 15 14
151532024-02-07T19:01:13.349Z20/30 Write block 21 len 40960 data: 15 15 17 14 13 11 11 13 17 16
151542024-02-07T19:01:13.349Z21/30 Read block 565 len 24576
151552024-02-07T19:01:13.353Z22/30 Flush
151562024-02-07T19:01:13.353Z23/30 Flush
151572024-02-07T19:01:13.353Z24/30 Write block 312 len 8192 data: 8 10
151582024-02-07T19:01:13.353Z25/30 Write block 473 len 36864 data: 8 8 9 10 9 8 8 8 7
151592024-02-07T19:01:13.353Z26/30 Write block 92 len 24576 data: 10 10 11 14 15 13
151602024-02-07T19:01:13.357Z27/30 Write block 91 len 16384 data: 8 11 11 12
151612024-02-07T19:01:13.357Z28/30 Read block 462 len 16384
151622024-02-07T19:01:13.357Z29/30 Write block 173 len 40960 data: 11 9 9 8 9 9 9 10 5 5
151632024-02-07T19:01:13.357Z30/30 Write block 366 len 8192 data: 11 11
151642024-02-07T19:01:13.358Z----------------------------------------------------------------
151652024-02-07T19:01:13.358Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
151662024-02-07T19:01:13.358ZStates: Active Active Active
151672024-02-07T19:01:13.358ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151682024-02-07T19:01:13.358Z 1 Acked 1000 Write 10 Done Sent Done false
151692024-02-07T19:01:13.358Z 2 Acked 1001 Write 10 Done Sent Done false
151702024-02-07T19:01:13.358Z 3 Acked 1002 Read 9 Done Sent Done false
151712024-02-07T19:01:13.358Z 4 Acked 1003 Read 10 Done Sent Done false
151722024-02-07T19:01:13.358Z 5 Acked 1004 Write 1 Done Sent Done false
151732024-02-07T19:01:13.358Z 6 Acked 1005 Write 8 Done Sent Done false
151742024-02-07T19:01:13.358Z 7 Acked 1006 Write 10 Done Sent Done false
151752024-02-07T19:01:13.358Z 8 Acked 1007 Read 4 Done Sent Done false
151762024-02-07T19:01:13.359Z 9 Acked 1008 Write 8 Done Sent Done false
151772024-02-07T19:01:13.359Z 10 Acked 1009 Read 2 Done Sent Done false
151782024-02-07T19:01:13.359Z 11 Acked 1010 Write 2 Done Sent Done false
151792024-02-07T19:01:13.359Z 12 Acked 1011 Read 7 Done Sent Done false
151802024-02-07T19:01:13.359Z 13 Acked 1012 Read 4 Done Sent Done false
151812024-02-07T19:01:13.359Z 14 Acked 1013 Write 10 Done Sent Done false
151822024-02-07T19:01:13.359Z 15 Acked 1014 Read 2 Done Sent Done false
151832024-02-07T19:01:13.359Z 16 Acked 1015 Read 10 Done Sent Done false
151842024-02-07T19:01:13.359Z 17 Acked 1016 Write 7 Done Sent Done false
151852024-02-07T19:01:13.360Z 18 Acked 1017 Flush 0 Done Sent Done false
151862024-02-07T19:01:13.360Z 19 Acked 1018 Write 9 Done Sent Done false
151872024-02-07T19:01:13.360Z 20 Acked 1019 Write 10 Done Sent Done false
151882024-02-07T19:01:13.360Z 21 Acked 1020 Read 6 Done Sent Done false
151892024-02-07T19:01:13.360Z 22 Acked 1021 Flush 0 Done Sent Done false
151902024-02-07T19:01:13.360Z 23 Acked 1022 Flush 0 Done Sent Done false
151912024-02-07T19:01:13.360Z 24 Acked 1023 Write 2 Done Sent Done false
151922024-02-07T19:01:13.360Z 25 Acked 1024 Write 9 Done Sent Done false
151932024-02-07T19:01:13.360Z 26 Acked 1025 Write 6 Done Sent Done false
151942024-02-07T19:01:13.360Z 27 Acked 1026 Write 4 Done Sent Done false
151952024-02-07T19:01:13.360Z 28 Acked 1027 Read 4 Done Sent Done false
151962024-02-07T19:01:13.360Z 29 Acked 1028 Write 10 Sent Sent Sent false
151972024-02-07T19:01:13.361Z 30 Acked 1029 Write 2 Sent Sent Sent false
151982024-02-07T19:01:13.361Z STATES DS:0 DS:1 DS:2 TOTAL
151992024-02-07T19:01:13.361Z New 0 0 0 0
152002024-02-07T19:01:13.361Z Sent 2 30 2 34
152012024-02-07T19:01:13.361Z Done 28 0 28 56
152022024-02-07T19:01:13.361Z Skipped 0 0 0 0
152032024-02-07T19:01:13.361Z Error 0 0 0 0
152042024-02-07T19:01:13.361ZLast Flush: 1022 0 1022
152052024-02-07T19:01:13.361ZDownstairs last five completed:
152062024-02-07T19:01:13.361ZUpstairs last five completed: 30 29 28 27 26
152072024-02-07T19:01:13.362Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:12.570845227Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1687}
152082024-02-07T19:01:13.362ZWrote out file "/tmp/test_repair_verify.data"
152092024-02-07T19:01:13.372Z
152102024-02-07T19:01:13.372ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
152112024-02-07T19:01:13.383ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152122024-02-07T19:01:13.383Z 0 000-019 132 132 132 229 229 229 F F F
152132024-02-07T19:01:13.383Z 1 020-039 138 134 138 236 231 236 F F F <---
152142024-02-07T19:01:13.383Z 2 040-059 134 134 134 231 231 231 F F F
152152024-02-07T19:01:13.383Z 3 060-079 138 137 138 235 234 235 F F F <---
152162024-02-07T19:01:13.383Z 4 080-099 138 136 138 235 232 235 T F T <---
152172024-02-07T19:01:13.383Z 5 100-119 137 137 137 234 234 234 F F F
152182024-02-07T19:01:13.383Z 6 120-139 137 137 137 234 234 234 F F F
152192024-02-07T19:01:13.384Z 7 140-159 138 128 138 235 223 235 F F F <---
152202024-02-07T19:01:13.384Z 8 160-179 138 131 138 235 228 235 T F T <---
152212024-02-07T19:01:13.384Z 9 180-199 138 134 138 235 231 235 T F T <---
152222024-02-07T19:01:13.388Z 10 200-219 138 137 138 235 234 235 F F F <---
152232024-02-07T19:01:13.388Z 11 220-239 138 136 138 235 232 235 F F F <---
152242024-02-07T19:01:13.388Z 12 240-259 138 137 138 235 234 235 F F F <---
152252024-02-07T19:01:13.388Z 13 260-279 132 132 132 229 229 229 F F F
152262024-02-07T19:01:13.388Z 14 280-299 134 134 134 231 231 231 F F F
152272024-02-07T19:01:13.388Z 15 300-319 136 136 136 232 232 232 T F T <---
152282024-02-07T19:01:13.388Z 16 320-339 137 137 137 234 234 234 F F F
152292024-02-07T19:01:13.389Z 17 340-359 138 132 138 235 229 235 F F F <---
152302024-02-07T19:01:13.389Z 18 360-379 138 136 138 235 233 235 T F T <---
152312024-02-07T19:01:13.389Z 19 380-399 136 136 136 233 233 233 F F F
152322024-02-07T19:01:13.389Z 20 400-419 136 136 136 233 233 233 F F F
152332024-02-07T19:01:13.389Z 21 420-439 137 137 137 234 234 234 F F F
152342024-02-07T19:01:13.389Z 22 440-459 130 130 130 224 224 224 F F F
152352024-02-07T19:01:13.389Z 23 460-479 136 136 136 232 232 232 T F T <---
152362024-02-07T19:01:13.389Z 24 480-499 138 137 138 235 234 235 T F T <---
152372024-02-07T19:01:13.389Z 25 500-519 136 136 136 232 232 232 F F F
152382024-02-07T19:01:13.389Z 26 520-539 138 136 138 235 232 235 F F F <---
152392024-02-07T19:01:13.390Z 27 540-559 134 134 134 231 231 231 F F F
152402024-02-07T19:01:13.390Z 28 560-579 137 137 137 234 234 234 F F F
152412024-02-07T19:01:13.390Z 29 580-599 137 137 137 234 234 234 F F F
152422024-02-07T19:01:13.390ZMax gen: 138, Max flush: 236
152432024-02-07T19:01:13.390ZError: Difference in extent metadata found!
152442024-02-07T19:01:13.390ZA Difference in extent metadata is expected here
152452024-02-07T19:01:13.391ZOn loop 69
152462024-02-07T19:01:13.391Z
152472024-02-07T19:01:13.395ZVerifying data now
152482024-02-07T19:01:19.094ZNo change after verify
152492024-02-07T19:01:19.095ZLoop: 69 Downstairs dump after verify (and repair):
152502024-02-07T19:01:19.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152512024-02-07T19:01:19.110Z 0 000-019 132 132 132 229 229 229 F F F
152522024-02-07T19:01:19.110Z 1 020-039 138 138 138 236 236 236 F F F
152532024-02-07T19:01:19.110Z 2 040-059 134 134 134 231 231 231 F F F
152542024-02-07T19:01:19.110Z 3 060-079 138 138 138 235 235 235 F F F
152552024-02-07T19:01:19.110Z 4 080-099 139 139 139 237 237 237 F F F
152562024-02-07T19:01:19.110Z 5 100-119 137 137 137 234 234 234 F F F
152572024-02-07T19:01:19.110Z 6 120-139 137 137 137 234 234 234 F F F
152582024-02-07T19:01:19.110Z 7 140-159 138 138 138 235 235 235 F F F
152592024-02-07T19:01:19.110Z 8 160-179 139 139 139 237 237 237 F F F
152602024-02-07T19:01:19.111Z 9 180-199 139 139 139 237 237 237 F F F
152612024-02-07T19:01:19.115Z 10 200-219 138 138 138 235 235 235 F F F
152622024-02-07T19:01:19.115Z 11 220-239 138 138 138 235 235 235 F F F
152632024-02-07T19:01:19.115Z 12 240-259 138 138 138 235 235 235 F F F
152642024-02-07T19:01:19.115Z 13 260-279 132 132 132 229 229 229 F F F
152652024-02-07T19:01:19.116Z 14 280-299 134 134 134 231 231 231 F F F
152662024-02-07T19:01:19.116Z 15 300-319 139 139 139 237 237 237 F F F
152672024-02-07T19:01:19.116Z 16 320-339 137 137 137 234 234 234 F F F
152682024-02-07T19:01:19.116Z 17 340-359 138 138 138 235 235 235 F F F
152692024-02-07T19:01:19.116Z 18 360-379 139 139 139 237 237 237 F F F
152702024-02-07T19:01:19.116Z 19 380-399 136 136 136 233 233 233 F F F
152712024-02-07T19:01:19.116Z 20 400-419 136 136 136 233 233 233 F F F
152722024-02-07T19:01:19.116Z 21 420-439 137 137 137 234 234 234 F F F
152732024-02-07T19:01:19.116Z 22 440-459 130 130 130 224 224 224 F F F
152742024-02-07T19:01:19.116Z 23 460-479 139 139 139 237 237 237 F F F
152752024-02-07T19:01:19.116Z 24 480-499 139 139 139 237 237 237 F F F
152762024-02-07T19:01:19.116Z 25 500-519 136 136 136 232 232 232 F F F
152772024-02-07T19:01:19.117Z 26 520-539 138 138 138 235 235 235 F F F
152782024-02-07T19:01:19.117Z 27 540-559 134 134 134 231 231 231 F F F
152792024-02-07T19:01:19.117Z 28 560-579 137 137 137 234 234 234 F F F
152802024-02-07T19:01:19.117Z 29 580-599 137 137 137 234 234 234 F F F
152812024-02-07T19:01:19.117ZMax gen: 139, Max flush: 237
152822024-02-07T19:01:19.117Z
152832024-02-07T19:01:19.117ZBegin loop 70
152842024-02-07T19:01:19.117ZDownstairs to restart: 1
152852024-02-07T19:01:19.131Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344381453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695}
152862024-02-07T19:01:19.131Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344636518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695}
152872024-02-07T19:01:19.131Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344663974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695}
152882024-02-07T19:01:19.131Z{"msg":"Crucible c1caf6b1-4388-4541-9be0-db0dc17ecf00 has session id: 115484b7-ef38-4d9e-885b-c36e00a3ab60","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
152892024-02-07T19:01:19.138Ztime":"2024-02-07T19:01:18.3446959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152902024-02-07T19:01:19.138Z{"msg":"Upstairs opts: Upstairs UUID: c1caf6b1-4388-4541-9be0-db0dc17ecf00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344784588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152912024-02-07T19:01:19.138Z{"msg":"Crucible stats registered with UUID: c1caf6b1-4388-4541-9be0-db0dc17ecf00","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344838581Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152922024-02-07T19:01:19.139Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344872186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695}
152932024-02-07T19:01:19.139Z{"msg":"c1caf6b1-4388-4541-9be0-db0dc17ecf00 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344894483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152942024-02-07T19:01:19.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344922489Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152952024-02-07T19:01:19.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344955355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152962024-02-07T19:01:19.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.344975272Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152972024-02-07T19:01:19.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345018926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152982024-02-07T19:01:19.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345049312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
152992024-02-07T19:01:19.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345074279Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153002024-02-07T19:01:19.140Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345106105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153012024-02-07T19:01:19.140Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345131361Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153022024-02-07T19:01:19.140Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345165876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153032024-02-07T19:01:19.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345192613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153042024-02-07T19:01:19.140Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:18.345220429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153052024-02-07T19:01:19.140Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:18.345248645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153062024-02-07T19:01:19.140Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:18.34528906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153072024-02-07T19:01:19.140Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345304998Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153082024-02-07T19:01:19.141Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345325695Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153092024-02-07T19:01:19.141Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.3453644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153102024-02-07T19:01:19.141Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345492142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153112024-02-07T19:01:19.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.345964008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153122024-02-07T19:01:19.141Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.346016792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153132024-02-07T19:01:19.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.346054416Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153142024-02-07T19:01:19.141Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.346091071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153152024-02-07T19:01:19.141Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.346579715Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153162024-02-07T19:01:19.142Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.346673483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153172024-02-07T19:01:19.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.347082787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153182024-02-07T19:01:19.142Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.347119612Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153192024-02-07T19:01:19.142Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.347146609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153202024-02-07T19:01:19.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.347179594Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153212024-02-07T19:01:19.142Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.347200711Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153222024-02-07T19:01:19.142Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:18.347230457Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153232024-02-07T19:01:20.142Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.346771079Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153242024-02-07T19:01:20.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.346969492Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153252024-02-07T19:01:20.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.34772682Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153262024-02-07T19:01:20.581Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.347766974Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153272024-02-07T19:01:20.581Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.348291983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153282024-02-07T19:01:20.581Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.34891353Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153292024-02-07T19:01:20.582Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.348965902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153302024-02-07T19:01:20.582Z{"msg":"[0]R flush_numbers[0..12]: [229, 236, 231, 235, 237, 234, 234, 235, 237, 237, 235, 235]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.348988249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153312024-02-07T19:01:20.582Z{"msg":"[0]R generation[0..12]: [132, 138, 134, 138, 139, 137, 137, 138, 139, 139, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349017356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153322024-02-07T19:01:20.582Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349037363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153332024-02-07T19:01:20.582Z{"msg":"[1]R flush_numbers[0..12]: [229, 236, 231, 235, 237, 234, 234, 235, 237, 237, 235, 235]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349065019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153342024-02-07T19:01:20.582Z{"msg":"[1]R generation[0..12]: [132, 138, 134, 138, 139, 137, 137, 138, 139, 139, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349083876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153352024-02-07T19:01:20.582Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349110103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153362024-02-07T19:01:20.582Z{"msg":"[2]R flush_numbers[0..12]: [229, 236, 231, 235, 237, 234, 234, 235, 237, 237, 235, 235]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.34912857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153372024-02-07T19:01:20.583Z{"msg":"[2]R generation[0..12]: [132, 138, 134, 138, 139, 137, 137, 138, 139, 139, 138, 138]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
153382024-02-07T19:01:20.583Z,"time":"2024-02-07T19:01:19.349154917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153392024-02-07T19:01:20.583Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349173814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153402024-02-07T19:01:20.583Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349199121Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153412024-02-07T19:01:20.583Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349216929Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153422024-02-07T19:01:20.583Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349234276Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153432024-02-07T19:01:20.583Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349252174Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153442024-02-07T19:01:20.584Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.34927733Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153452024-02-07T19:01:20.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349293198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153462024-02-07T19:01:20.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349325794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153472024-02-07T19:01:20.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349342801Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153482024-02-07T19:01:20.584Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349359919Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153492024-02-07T19:01:20.585Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349375667Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153502024-02-07T19:01:20.585Z{"msg":"c1caf6b1-4388-4541-9be0-db0dc17ecf00 is now active with session: 115484b7-ef38-4d9e-885b-c36e00a3ab60","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349401784Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153512024-02-07T19:01:20.585Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349417521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695,"session_id":"115484b7-ef38-4d9e-885b-c36e00a3ab60"}
153522024-02-07T19:01:20.585Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:19.349448247Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695}
153532024-02-07T19:01:20.585ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153542024-02-07T19:01:20.585ZLoading write count information from file "/tmp/test_repair_verify.data"
153552024-02-07T19:01:20.586ZRun Repair workload
153562024-02-07T19:01:20.586Z01/30 Write block 275 len 28672 data: 6 4 5 7 6 7 8
153572024-02-07T19:01:20.586Z02/30 Write block 230 len 4096 data: 9
153582024-02-07T19:01:20.586Z03/30 Write block 428 len 32768 data: 12 11 11 12 10 8 6 7
153592024-02-07T19:01:20.586Z04/30 Flush
153602024-02-07T19:01:20.586Z05/30 Write block 394 len 24576 data: 9 8 9 9 12 15
153612024-02-07T19:01:20.586Z06/30 Read block 273 len 20480
153622024-02-07T19:01:20.586Z07/30 Write block 545 len 16384 data: 6 8 13 14
153632024-02-07T19:01:20.587Z08/30 Read block 368 len 24576
153642024-02-07T19:01:20.587Z09/30 Read block 133 len 20480
153652024-02-07T19:01:20.587Z10/30 Read block 389 len 24576
153662024-02-07T19:01:20.587Z11/30 Read block 355 len 24576
153672024-02-07T19:01:20.587Z12/30 Write block 184 len 20480 data: 4 6 6 8 10
153682024-02-07T19:01:20.587Z13/30 Read block 295 len 4096
153692024-02-07T19:01:20.587Z14/30 Write block 194 len 32768 data: 14 11 10 11 13 12 11 11
153702024-02-07T19:01:20.588Z15/30 Read block 376 len 40960
153712024-02-07T19:01:20.588Z16/30 Read block 479 len 24576
153722024-02-07T19:01:20.588Z17/30 Read block 115 len 40960
153732024-02-07T19:01:20.588Z18/30 Write block 168 len 4096 data: 14
153742024-02-07T19:01:20.588Z19/30 Read block 307 len 32768
153752024-02-07T19:01:20.588Z20/30 Write block 479 len 12288 data: 9 9 8
153762024-02-07T19:01:20.589Z21/30 Write block 531 len 28672 data: 8 9 9 10 13 13 14
153772024-02-07T19:01:20.589Z22/30 Read block 450 len 32768
153782024-02-07T19:01:20.589Z23/30 Read block 588 len 16384
153792024-02-07T19:01:20.589Z24/30 Write block 76 len 4096 data: 8
153802024-02-07T19:01:20.589Z25/30 Write block 105 len 24576 data: 17 16 16 15 14 11
153812024-02-07T19:01:20.589Z26/30 Write block 123 len 16384 data: 14 15 15 14
153822024-02-07T19:01:20.590Z27/30 Read block 535 len 8192
153832024-02-07T19:01:20.590Z28/30 Write block 222 len 36864 data: 7 11 12 10 7 7 8 10 10
153842024-02-07T19:01:20.590Z29/30 Write block 511 len 32768 data: 11 15 12 15 14 13 10 12
153852024-02-07T19:01:20.590Z30/30 Write block 531 len 20480 data: 9 10 10 11 14
153862024-02-07T19:01:20.590Z----------------------------------------------------------------
153872024-02-07T19:01:20.590Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
153882024-02-07T19:01:20.590ZStates: Active Active Active
153892024-02-07T19:01:20.591ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153902024-02-07T19:01:20.591Z 1 Acked 1000 Write 7 Done Done Done false
153912024-02-07T19:01:20.591Z 2 Acked 1001 Write 1 Done Done Done false
153922024-02-07T19:01:20.591Z 3 Acked 1002 Write 8 Done Sent Done false
153932024-02-07T19:01:20.591Z 4 Acked 1003 Flush 0 Done Sent Done false
153942024-02-07T19:01:20.591Z 5 Acked 1004 Write 6 Done Sent Done false
153952024-02-07T19:01:20.591Z 6 Acked 1005 Read 5 Done Sent Done false
153962024-02-07T19:01:20.592Z 7 Acked 1006 Write 4 Done Sent Done false
153972024-02-07T19:01:20.592Z 8 Acked 1007 Read 6 Done Sent Done false
153982024-02-07T19:01:20.592Z 9 Acked 1008 Read 5 Done Sent Done false
153992024-02-07T19:01:20.592Z 10 Acked 1009 Read 6 Done Sent Done false
154002024-02-07T19:01:20.592Z 11 Acked 1010 Read 6 Done Sent Done false
154012024-02-07T19:01:20.592Z 12 Acked 1011 Write 5 Done Sent Done false
154022024-02-07T19:01:20.592Z 13 Acked 1012 Read 1 Done Sent Done false
154032024-02-07T19:01:20.592Z 14 Acked 1013 Write 8 Done Sent Done false
154042024-02-07T19:01:20.593Z 15 Acked 1014 Read 10 Done Sent Done false
154052024-02-07T19:01:20.593Z 16 Acked 1015 Read 6 Done Sent Done false
154062024-02-07T19:01:20.593Z 17 Acked 1016 Read 10 Done Sent Done false
154072024-02-07T19:01:20.593Z 18 Acked 1017 Write 1 Done Sent Done false
154082024-02-07T19:01:20.593Z 19 Acked 1018 Read 8 Done Sent Done false
154092024-02-07T19:01:20.593Z 20 Acked 1019 Write 3 Done Sent Done false
154102024-02-07T19:01:20.593Z 21 Acked 1020 Write 7 Done Sent Done false
154112024-02-07T19:01:20.594Z 22 Acked 1021 Read 8 Done Sent Done false
154122024-02-07T19:01:20.594Z 23 Acked 1022 Read 4 Done Sent Done false
154132024-02-07T19:01:20.594Z 24 Acked 1023 Write 1 Done Sent Done false
154142024-02-07T19:01:20.594Z 25 Acked 1024 Write 6 Done Sent Done false
154152024-02-07T19:01:20.594Z 26 Acked 1025 Write 4 Done Sent Done false
154162024-02-07T19:01:20.594Z 27 Acked 1026 Read 2 Done Sent Done false
154172024-02-07T19:01:20.594Z 28 Acked 1027 Write 9 Done Sent Done false
154182024-02-07T19:01:20.595Z 29 Acked 1028 Write 8 Sent Sent Sent false
154192024-02-07T19:01:20.595Z 30 Acked 1029 Write 5 Sent Sent Sent false
154202024-02-07T19:01:20.595Z STATES DS:0 DS:1 DS:2 TOTAL
154212024-02-07T19:01:20.595Z New 0 0 0 0
154222024-02-07T19:01:20.595Z Sent 2 28 2 32
154232024-02-07T19:01:20.595Z Done 28 2 28 58
154242024-02-07T19:01:20.608Z Skipped 0 0 0 0
154252024-02-07T19:01:20.608Z Error 0 0 0 0
154262024-02-07T19:01:20.608ZLast Flush: 1003 0 1003
154272024-02-07T19:01:20.608ZDownstairs last five completed:
154282024-02-07T19:01:20.608ZUpstairs last five completed: 30 29 28 27 26
154292024-02-07T19:01:20.608Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:19.377712814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1695}
154302024-02-07T19:01:20.608ZWrote out file "/tmp/test_repair_verify.data"
154312024-02-07T19:01:20.608Z
154322024-02-07T19:01:20.608ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
154332024-02-07T19:01:20.609ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154342024-02-07T19:01:20.609Z 0 000-019 132 132 132 229 229 229 F F F
154352024-02-07T19:01:20.609Z 1 020-039 138 138 138 236 236 236 F F F
154362024-02-07T19:01:20.609Z 2 040-059 134 134 134 231 231 231 F F F
154372024-02-07T19:01:20.609Z 3 060-079 138 138 138 235 235 235 T F T <---
154382024-02-07T19:01:20.609Z 4 080-099 139 139 139 237 237 237 F F F
154392024-02-07T19:01:20.609Z 5 100-119 137 137 137 234 234 234 T F T <---
154402024-02-07T19:01:20.609Z 6 120-139 137 137 137 234 234 234 T F T <---
154412024-02-07T19:01:20.609Z 7 140-159 138 138 138 235 235 235 F F F
154422024-02-07T19:01:20.610Z 8 160-179 139 139 139 237 237 237 T F T <---
154432024-02-07T19:01:20.610Z 9 180-199 139 139 139 237 237 237 T F T <---
154442024-02-07T19:01:20.610Z 10 200-219 138 138 138 235 235 235 T F T <---
154452024-02-07T19:01:20.610Z 11 220-239 140 138 140 238 235 238 T T T <---
154462024-02-07T19:01:20.610Z 12 240-259 138 138 138 235 235 235 F F F
154472024-02-07T19:01:20.610Z 13 260-279 140 132 140 238 229 238 F T F <---
154482024-02-07T19:01:20.610Z 14 280-299 140 134 140 238 231 238 F T F <---
154492024-02-07T19:01:20.610Z 15 300-319 139 139 139 237 237 237 F F F
154502024-02-07T19:01:20.610Z 16 320-339 137 137 137 234 234 234 F F F
154512024-02-07T19:01:20.610Z 17 340-359 138 138 138 235 235 235 F F F
154522024-02-07T19:01:20.611Z 18 360-379 139 139 139 237 237 237 F F F
154532024-02-07T19:01:20.611Z 19 380-399 136 136 136 233 233 233 T F T <---
154542024-02-07T19:01:20.611Z 20 400-419 136 136 136 233 233 233 F F F
154552024-02-07T19:01:20.611Z 21 420-439 140 137 140 238 234 238 F F F <---
154562024-02-07T19:01:20.611Z 22 440-459 130 130 130 224 224 224 F F F
154572024-02-07T19:01:20.611Z 23 460-479 139 139 139 237 237 237 T F T <---
154582024-02-07T19:01:20.611Z 24 480-499 139 139 139 237 237 237 T F T <---
154592024-02-07T19:01:20.611Z 25 500-519 136 136 136 232 232 232 T F T <---
154602024-02-07T19:01:20.611Z 26 520-539 138 138 138 235 235 235 T F T <---
154612024-02-07T19:01:20.612Z 27 540-559 134 134 134 231 231 231 T F T <---
154622024-02-07T19:01:20.612Z 28 560-579 137 137 137 234 234 234 F F F
154632024-02-07T19:01:20.612Z 29 580-599 137 137 137 234 234 234 F F F
154642024-02-07T19:01:20.612ZMax gen: 140, Max flush: 238
154652024-02-07T19:01:20.612ZError: Difference in extent metadata found!
154662024-02-07T19:01:20.612ZA Difference in extent metadata is expected here
154672024-02-07T19:01:20.612ZOn loop 70
154682024-02-07T19:01:20.613Z
154692024-02-07T19:01:20.613ZVerifying data now
154702024-02-07T19:01:26.087ZNo change after verify
154712024-02-07T19:01:26.088ZLoop: 70 Downstairs dump after verify (and repair):
154722024-02-07T19:01:26.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154732024-02-07T19:01:26.107Z 0 000-019 132 132 132 229 229 229 F F F
154742024-02-07T19:01:26.107Z 1 020-039 138 138 138 236 236 236 F F F
154752024-02-07T19:01:26.107Z 2 040-059 134 134 134 231 231 231 F F F
154762024-02-07T19:01:26.107Z 3 060-079 141 141 141 239 239 239 F F F
154772024-02-07T19:01:26.107Z 4 080-099 139 139 139 237 237 237 F F F
154782024-02-07T19:01:26.107Z 5 100-119 141 141 141 239 239 239 F F F
154792024-02-07T19:01:26.107Z 6 120-139 141 141 141 239 239 239 F F F
154802024-02-07T19:01:26.107Z 7 140-159 138 138 138 235 235 235 F F F
154812024-02-07T19:01:26.107Z 8 160-179 141 141 141 239 239 239 F F F
154822024-02-07T19:01:26.108Z 9 180-199 141 141 141 239 239 239 F F F
154832024-02-07T19:01:26.108Z 10 200-219 141 141 141 239 239 239 F F F
154842024-02-07T19:01:26.108Z 11 220-239 141 141 141 239 239 239 F F F
154852024-02-07T19:01:26.108Z 12 240-259 138 138 138 235 235 235 F F F
154862024-02-07T19:01:26.108Z 13 260-279 140 140 140 238 238 238 F F F
154872024-02-07T19:01:26.108Z 14 280-299 140 140 140 238 238 238 F F F
154882024-02-07T19:01:26.108Z 15 300-319 139 139 139 237 237 237 F F F
154892024-02-07T19:01:26.108Z 16 320-339 137 137 137 234 234 234 F F F
154902024-02-07T19:01:26.108Z 17 340-359 138 138 138 235 235 235 F F F
154912024-02-07T19:01:26.108Z 18 360-379 139 139 139 237 237 237 F F F
154922024-02-07T19:01:26.108Z 19 380-399 141 141 141 239 239 239 F F F
154932024-02-07T19:01:26.108Z 20 400-419 136 136 136 233 233 233 F F F
154942024-02-07T19:01:26.109Z 21 420-439 140 140 140 238 238 238 F F F
154952024-02-07T19:01:26.109Z 22 440-459 130 130 130 224 224 224 F F F
154962024-02-07T19:01:26.109Z 23 460-479 141 141 141 239 239 239 F F F
154972024-02-07T19:01:26.109Z 24 480-499 141 141 141 239 239 239 F F F
154982024-02-07T19:01:26.109Z 25 500-519 141 141 141 239 239 239 F F F
154992024-02-07T19:01:26.109Z 26 520-539 141 141 141 239 239 239 F F F
155002024-02-07T19:01:26.109Z 27 540-559 141 141 141 239 239 239 F F F
155012024-02-07T19:01:26.109Z 28 560-579 137 137 137 234 234 234 F F F
155022024-02-07T19:01:26.109Z 29 580-599 137 137 137 234 234 234 F F F
155032024-02-07T19:01:26.109ZMax gen: 141, Max flush: 239
155042024-02-07T19:01:26.109Z
155052024-02-07T19:01:26.109ZBegin loop 71
155062024-02-07T19:01:26.109ZDownstairs to restart: 2
155072024-02-07T19:01:26.110Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.200881207Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703}
155082024-02-07T19:01:26.110Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201115195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703}
155092024-02-07T19:01:26.110Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201142811Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703}
155102024-02-07T19:01:26.110Z{"msg":"Crucible c8bc3bf7-6006-4d7a-ba1b-79570e9d98d4 has session id: 3190f7a1-55d7-4f7c-a2d3-39349440c877","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201185926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155112024-02-07T19:01:26.110Z{"msg":"Upstairs opts: Upstairs UUID: c8bc3bf7-6006-4d7a-ba1b-79570e9d98d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201222461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155122024-02-07T19:01:26.110ZCrucible runtime is spawned
155132024-02-07T19:01:26.110Z{"msg":"Crucible stats registered with UUID: c8bc3bf7-6006-4d7a-ba1b-79570e9d98d4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201273794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155142024-02-07T19:01:26.110Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201342844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703}
155152024-02-07T19:01:26.110Z{"msg":"c8bc3bf7-6006-4d7a-ba1b-79570e9d98d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201409265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155162024-02-07T19:01:26.110Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201459369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155172024-02-07T19:01:26.110Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201490654Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155182024-02-07T19:01:26.110Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201530839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155192024-02-07T19:01:26.110Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201560205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155202024-02-07T19:01:26.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.20159927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155212024-02-07T19:01:26.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201628975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155222024-02-07T19:01:26.111Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201655162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155232024-02-07T19:01:26.111Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201695257Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155242024-02-07T19:01:26.111Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201733001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155252024-02-07T19:01:26.111Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201767147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155262024-02-07T19:01:26.111Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:25.201875172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155272024-02-07T19:01:26.111Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:25.201919716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155282024-02-07T19:01:26.111Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.201936814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155292024-02-07T19:01:26.111Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:25.201973679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155302024-02-07T19:01:26.111Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202023382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155312024-02-07T19:01:26.111Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202060277Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155322024-02-07T19:01:26.111Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202163123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155332024-02-07T19:01:26.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202552841Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155342024-02-07T19:01:26.112Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202587736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155352024-02-07T19:01:26.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202649037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155362024-02-07T19:01:26.112Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.202671574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155372024-02-07T19:01:26.112Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203027396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155382024-02-07T19:01:26.112Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203139221Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155392024-02-07T19:01:26.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203571893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155402024-02-07T19:01:26.112Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203609987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155412024-02-07T19:01:26.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203647022Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155422024-02-07T19:01:26.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203673489Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155432024-02-07T19:01:26.112Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203704595Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155442024-02-07T19:01:26.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:25.203729282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155452024-02-07T19:01:27.097Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.202586026Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155462024-02-07T19:01:27.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.2027055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155472024-02-07T19:01:27.111Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.203561934Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155482024-02-07T19:01:27.111Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.203601009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155492024-02-07T19:01:27.111Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204148044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155502024-02-07T19:01:27.111Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204690981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155512024-02-07T19:01:27.111Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204732116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155522024-02-07T19:01:27.111Z{"msg":"[0]R flush_numbers[0..12]: [229, 236, 231, 239, 237, 239, 239, 235, 239, 239, 239, 239]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204753263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155532024-02-07T19:01:27.111Z{"msg":"[0]R generation[0..12]: [132, 138, 134, 141, 139, 141, 141, 138, 141, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.20477222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155542024-02-07T19:01:27.111Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204791097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155552024-02-07T19:01:27.111Z{"msg":"[1]R flush_numbers[0..12]: [229, 236, 231, 239, 237, 239, 239, 235, 239, 239, 239, 239]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204818464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155562024-02-07T19:01:27.112Z{"msg":"[1]R generation[0..12]: [132, 138, 134, 141, 139, 141, 141, 138, 141, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204836611Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155572024-02-07T19:01:27.112Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204862508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155582024-02-07T19:01:27.112Z{"msg":"[2]R flush_numbers[0..12]: [229, 236, 231, 239, 237, 239, 239, 235, 239, 239, 239, 239]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204880756Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155592024-02-07T19:01:27.112Z{Wait for a query_work_queue command to finish before sending IO
155602024-02-07T19:01:27.112Z"msg":"[2]R generation[0..12]: [132, 138, 134, 141, 139, 141, 141, 138, 141, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204915681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155612024-02-07T19:01:27.112Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204946147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155622024-02-07T19:01:27.112Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204970063Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155632024-02-07T19:01:27.112Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.204987611Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155642024-02-07T19:01:27.112Z{"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205023616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155652024-02-07T19:01:27.112Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205055292Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155662024-02-07T19:01:27.112Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205093787Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155672024-02-07T19:01:27.112Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205135231Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155682024-02-07T19:01:27.112Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205173586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155692024-02-07T19:01:27.112Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.2052135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155702024-02-07T19:01:27.112Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205253705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155712024-02-07T19:01:27.112Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205294849Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155722024-02-07T19:01:27.112Zip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155732024-02-07T19:01:27.112Z{"msg":"c8bc3bf7-6006-4d7a-ba1b-79570e9d98d4 is now active with session: 3190f7a1-55d7-4f7c-a2d3-39349440c877","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205339103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155742024-02-07T19:01:27.112Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205368439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703,"session_id":"3190f7a1-55d7-4f7c-a2d3-39349440c877"}
155752024-02-07T19:01:27.112Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:26.205385087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703}
155762024-02-07T19:01:27.112ZLoading write count information from file "/tmp/test_repair_verify.data"
155772024-02-07T19:01:27.112ZRun Repair workload
155782024-02-07T19:01:27.112Z01/30 Write block 507 len 16384 data: 11 12 12 14
155792024-02-07T19:01:27.113Z02/30 Write block 272 len 28672 data: 7 8 6 7 5 6 8
155802024-02-07T19:01:27.113Z03/30 Write block 551 len 32768 data: 11 9 9 8 10 9 7 9
155812024-02-07T19:01:27.113Z04/30 Read block 27 len 36864
155822024-02-07T19:01:27.113Z05/30 Write block 116 len 8192 data: 12 14
155832024-02-07T19:01:27.113Z06/30 Write block 18 len 40960 data: 12 15 16 16 16 18 15 14 12 12
155842024-02-07T19:01:27.113Z07/30 Read block 29 len 8192
155852024-02-07T19:01:27.113Z08/30 Read block 283 len 40960
155862024-02-07T19:01:27.113Z09/30 Read block 300 len 4096
155872024-02-07T19:01:27.113Z10/30 Read block 90 len 4096
155882024-02-07T19:01:27.113Z11/30 Read block 387 len 36864
155892024-02-07T19:01:27.113Z12/30 Read block 526 len 12288
155902024-02-07T19:01:27.113Z13/30 Write block 13 len 36864 data: 10 11 9 11 14 13 16 17 17
155912024-02-07T19:01:27.113Z14/30 Read block 121 len 4096
155922024-02-07T19:01:27.113Z15/30 Read block 116 len 4096
155932024-02-07T19:01:27.113Z16/30 Write block 158 len 20480 data: 10 9 9 10 11
155942024-02-07T19:01:27.113Z17/30 Read block 257 len 4096
155952024-02-07T19:01:27.113Z18/30 Read block 581 len 32768
155962024-02-07T19:01:27.113Z19/30 Read block 255 len 28672
155972024-02-07T19:01:27.113Z20/30 Write block 517 len 40960 data: 11 13 11 11 8 6 6 9 8 9
155982024-02-07T19:01:27.113Z21/30 Read block 285 len 24576
155992024-02-07T19:01:27.113Z22/30 Write block 492 len 4096 data: 10
156002024-02-07T19:01:27.113Z23/30 Write block 81 len 40960 data: 8 8 8 7 6 6 5 6 6 8
156012024-02-07T19:01:27.113Z24/30 Write block 361 len 32768 data: 11 13 12 9 8 12 12 12
156022024-02-07T19:01:27.113Z25/30 Write block 163 len 24576 data: 13 16 15 12 11 15
156032024-02-07T19:01:27.114Z26/30 Flush
156042024-02-07T19:01:27.114Z27/30 Write block 11 len 40960 data: 13 12 11 12 10 12 15 14 17 18
156052024-02-07T19:01:27.114Z28/30 Read block 552 len 32768
156062024-02-07T19:01:27.114Z29/30 Write block 67 len 24576 data: 12 12 11 10 11 9
156072024-02-07T19:01:27.114Z30/30 Write block 1 len 28672 data: 4 3 5 8 12 11 14
156082024-02-07T19:01:27.114Z----------------------------------------------------------------
156092024-02-07T19:01:27.114Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
156102024-02-07T19:01:27.114ZStates: Active Active Active
156112024-02-07T19:01:27.114ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156122024-02-07T19:01:27.114Z 1 Acked 1000 Write 4 Done Done Done false
156132024-02-07T19:01:27.114Z 2 Acked 1001 Write 7 Done Done Done false
156142024-02-07T19:01:27.114Z 3 Acked 1002 Write 8 Done Done Done false
156152024-02-07T19:01:27.114Z 4 Acked 1003 Read 9 Done Done Done false
156162024-02-07T19:01:27.114Z 5 Acked 1004 Write 2 Done Done Done false
156172024-02-07T19:01:27.114Z 6 Acked 1005 Write 10 Done Done Done false
156182024-02-07T19:01:27.114Z 7 Acked 1006 Read 2 Done Done Sent false
156192024-02-07T19:01:27.114Z 8 Acked 1007 Read 10 Done Done Sent false
156202024-02-07T19:01:27.114Z 9 Acked 1008 Read 1 Done Done Sent false
156212024-02-07T19:01:27.114Z 10 Acked 1009 Read 1 Done Done Sent false
156222024-02-07T19:01:27.114Z 11 Acked 1010 Read 9 Done Done Sent false
156232024-02-07T19:01:27.114Z 12 Acked 1011 Read 3 Done Done Sent false
156242024-02-07T19:01:27.114Z 13 Acked 1012 Write 9 Done Done Sent false
156252024-02-07T19:01:27.114Z 14 Acked 1013 Read 1 Done Done Sent false
156262024-02-07T19:01:27.114Z 15 Acked 1014 Read 1 Done Done Sent false
156272024-02-07T19:01:27.115Z 16 Acked 1015 Write 5 Done Done Sent false
156282024-02-07T19:01:27.115Z 17 Acked 1016 Read 1 Done Done Sent false
156292024-02-07T19:01:27.115Z 18 Acked 1017 Read 8 Done Done Sent false
156302024-02-07T19:01:27.115Z 19 Acked 1018 Read 7 Done Done Sent false
156312024-02-07T19:01:27.115Z 20 Acked 1019 Write 10 Done Done Sent false
156322024-02-07T19:01:27.115Z 21 Acked 1020 Read 6 Done Done Sent false
156332024-02-07T19:01:27.115Z 22 Acked 1021 Write 1 Done Done Sent false
156342024-02-07T19:01:27.115Z 23 Acked 1022 Write 10 Done Done Sent false
156352024-02-07T19:01:27.115Z 24 Acked 1023 Write 8 Done Done Sent false
156362024-02-07T19:01:27.115Z 25 Acked 1024 Write 6 Done Done Sent false
156372024-02-07T19:01:27.115Z 26 Acked 1025 Flush 0 Done Done Sent false
156382024-02-07T19:01:27.115Z 27 Acked 1026 Write 10 Done Done Sent false
156392024-02-07T19:01:27.115Z 28 Acked 1027 Read 8 Done Done Sent false
156402024-02-07T19:01:27.115Z 29 Acked 1028 Write 6 Done Done Sent false
156412024-02-07T19:01:27.115Z 30 Acked 1029 Write 7 Sent Sent Sent false
156422024-02-07T19:01:27.115Z STATES DS:0 DS:1 DS:2 TOTAL
156432024-02-07T19:01:27.115Z New 0 0 0 0
156442024-02-07T19:01:27.115Z Sent 1 1 24 26
156452024-02-07T19:01:27.115Z Done 29 29 6 64
156462024-02-07T19:01:27.115Z Skipped 0 0 0 0
156472024-02-07T19:01:27.119Z Error 0 0 0 0
156482024-02-07T19:01:27.119ZLast Flush: 1025 1025 0
156492024-02-07T19:01:27.120ZDownstairs last five completed:
156502024-02-07T19:01:27.120ZUpstairs last five completed: 30 29 28 27 26
156512024-02-07T19:01:27.120Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:26.236182072Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1703}
156522024-02-07T19:01:27.120ZWrote out file "/tmp/test_repair_verify.data"
156532024-02-07T19:01:27.120Z
156542024-02-07T19:01:27.120ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156552024-02-07T19:01:27.120ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156562024-02-07T19:01:27.120Z 0 000-019 142 142 132 240 240 229 T T T <---
156572024-02-07T19:01:27.120Z 1 020-039 142 142 138 240 240 236 T T T <---
156582024-02-07T19:01:27.120Z 2 040-059 134 134 134 231 231 231 F F F
156592024-02-07T19:01:27.120Z 3 060-079 141 141 141 239 239 239 T T F <---
156602024-02-07T19:01:27.120Z 4 080-099 142 142 139 240 240 237 F F F <---
156612024-02-07T19:01:27.120Z 5 100-119 142 142 141 240 240 239 F F T <---
156622024-02-07T19:01:27.120Z 6 120-139 141 141 141 239 239 239 F F F
156632024-02-07T19:01:27.120Z 7 140-159 142 142 138 240 240 235 F F F <---
156642024-02-07T19:01:27.120Z 8 160-179 142 142 141 240 240 239 F F F <---
156652024-02-07T19:01:27.120Z 9 180-199 141 141 141 239 239 239 F F F
156662024-02-07T19:01:27.120Z 10 200-219 141 141 141 239 239 239 F F F
156672024-02-07T19:01:27.120Z 11 220-239 141 141 141 239 239 239 F F F
156682024-02-07T19:01:27.120Z 12 240-259 138 138 138 235 235 235 F F F
156692024-02-07T19:01:27.120Z 13 260-279 142 142 140 240 240 238 F F T <---
156702024-02-07T19:01:27.120Z 14 280-299 140 140 140 238 238 238 F F F
156712024-02-07T19:01:27.120Z 15 300-319 139 139 139 237 237 237 F F F
156722024-02-07T19:01:27.121Z 16 320-339 137 137 137 234 234 234 F F F
156732024-02-07T19:01:27.121Z 17 340-359 138 138 138 235 235 235 F F F
156742024-02-07T19:01:27.121Z 18 360-379 142 142 139 240 240 237 F F F <---
156752024-02-07T19:01:27.121Z 19 380-399 141 141 141 239 239 239 F F F
156762024-02-07T19:01:27.121Z 20 400-419 136 136 136 233 233 233 F F F
156772024-02-07T19:01:27.121Z 21 420-439 140 140 140 238 238 238 F F F
156782024-02-07T19:01:27.121Z 22 440-459 130 130 130 224 224 224 F F F
156792024-02-07T19:01:27.121Z 23 460-479 141 141 141 239 239 239 F F F
156802024-02-07T19:01:27.121Z 24 480-499 142 142 141 240 240 239 F F F <---
156812024-02-07T19:01:27.121Z 25 500-519 142 142 141 240 240 239 F F T <---
156822024-02-07T19:01:27.121Z 26 520-539 142 142 141 240 240 239 F F F <---
156832024-02-07T19:01:27.121Z 27 540-559 142 142 141 240 240 239 F F T <---
156842024-02-07T19:01:27.121Z 28 560-579 137 137 137 234 234 234 F F F
156852024-02-07T19:01:27.121Z 29 580-599 137 137 137 234 234 234 F F F
156862024-02-07T19:01:27.121ZMax gen: 142, Max flush: 240
156872024-02-07T19:01:27.121ZError: Difference in extent metadata found!
156882024-02-07T19:01:27.121ZA Difference in extent metadata is expected here
156892024-02-07T19:01:27.121ZOn loop 71
156902024-02-07T19:01:27.121Z
156912024-02-07T19:01:27.121ZVerifying data now
156922024-02-07T19:01:32.678ZNo change after verify
156932024-02-07T19:01:32.678ZLoop: 71 Downstairs dump after verify (and repair):
156942024-02-07T19:01:32.694ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156952024-02-07T19:01:32.694Z 0 000-019 143 143 143 241 241 241 F F F
156962024-02-07T19:01:32.694Z 1 020-039 143 143 143 241 241 241 F F F
156972024-02-07T19:01:32.694Z 2 040-059 134 134 134 231 231 231 F F F
156982024-02-07T19:01:32.694Z 3 060-079 143 143 143 241 241 241 F F F
156992024-02-07T19:01:32.694Z 4 080-099 142 142 142 240 240 240 F F F
157002024-02-07T19:01:32.694Z 5 100-119 142 142 142 240 240 240 F F F
157012024-02-07T19:01:32.694Z 6 120-139 141 141 141 239 239 239 F F F
157022024-02-07T19:01:32.694Z 7 140-159 142 142 142 240 240 240 F F F
157032024-02-07T19:01:32.698Z 8 160-179 142 142 142 240 240 240 F F F
157042024-02-07T19:01:32.698Z 9 180-199 141 141 141 239 239 239 F F F
157052024-02-07T19:01:32.698Z 10 200-219 141 141 141 239 239 239 F F F
157062024-02-07T19:01:32.698Z 11 220-239 141 141 141 239 239 239 F F F
157072024-02-07T19:01:32.698Z 12 240-259 138 138 138 235 235 235 F F F
157082024-02-07T19:01:32.698Z 13 260-279 142 142 142 240 240 240 F F F
157092024-02-07T19:01:32.698Z 14 280-299 140 140 140 238 238 238 F F F
157102024-02-07T19:01:32.698Z 15 300-319 139 139 139 237 237 237 F F F
157112024-02-07T19:01:32.698Z 16 320-339 137 137 137 234 234 234 F F F
157122024-02-07T19:01:32.698Z 17 340-359 138 138 138 235 235 235 F F F
157132024-02-07T19:01:32.698Z 18 360-379 142 142 142 240 240 240 F F F
157142024-02-07T19:01:32.698Z 19 380-399 141 141 141 239 239 239 F F F
157152024-02-07T19:01:32.698Z 20 400-419 136 136 136 233 233 233 F F F
157162024-02-07T19:01:32.698Z 21 420-439 140 140 140 238 238 238 F F F
157172024-02-07T19:01:32.699Z 22 440-459 130 130 130 224 224 224 F F F
157182024-02-07T19:01:32.699Z 23 460-479 141 141 141 239 239 239 F F F
157192024-02-07T19:01:32.699Z 24 480-499 142 142 142 240 240 240 F F F
157202024-02-07T19:01:32.699Z 25 500-519 142 142 142 240 240 240 F F F
157212024-02-07T19:01:32.699Z 26 520-539 142 142 142 240 240 240 F F F
157222024-02-07T19:01:32.699Z 27 540-559 142 142 142 240 240 240 F F F
157232024-02-07T19:01:32.699Z 28 560-579 137 137 137 234 234 234 F F F
157242024-02-07T19:01:32.699Z 29 580-599 137 137 137 234 234 234 F F F
157252024-02-07T19:01:32.699ZMax gen: 143, Max flush: 241
157262024-02-07T19:01:32.699Z
157272024-02-07T19:01:32.699ZBegin loop 72
157282024-02-07T19:01:32.699ZDownstairs to restart: 1
157292024-02-07T19:01:32.717Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.929096762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711}
157302024-02-07T19:01:32.718Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.929390763Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711}
157312024-02-07T19:01:32.718Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.929421278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711}
157322024-02-07T19:01:32.722Z{"msg":"Crucible 8d0a2ae6-baf7-476d-92d7-22c3acd7eafc has session id: c4f08d27-8384-487c-a654-2d2822b5aa63","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.930788253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157332024-02-07T19:01:32.722Z{"msg":"Upstairs opts: Upstairs UUID: 8d0a2ae6-baf7-476d-92d7-22c3acd7eafc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.930833278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157342024-02-07T19:01:32.722Z{"msg":"Crucible stats registered with UUID: 8d0a2ae6-baf7-476d-92d7-22c3acd7eafc","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.930849405Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157352024-02-07T19:01:32.722ZCrucible runtime is spawned
157362024-02-07T19:01:32.722Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931139326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711}
157372024-02-07T19:01:32.722Z{"msg":"8d0a2ae6-baf7-476d-92d7-22c3acd7eafc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931233323Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157382024-02-07T19:01:32.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.93126254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157392024-02-07T19:01:32.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931309383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157402024-02-07T19:01:32.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931356217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157412024-02-07T19:01:32.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931385253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157422024-02-07T19:01:32.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931411689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157432024-02-07T19:01:32.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931436586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157442024-02-07T19:01:32.723Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931561309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157452024-02-07T19:01:32.723Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931612842Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157462024-02-07T19:01:32.723Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931655766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157472024-02-07T19:01:32.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.93170105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157482024-02-07T19:01:32.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931743375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157492024-02-07T19:01:32.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.931783359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157502024-02-07T19:01:32.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.932313857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157512024-02-07T19:01:32.723Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.93237046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157522024-02-07T19:01:32.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.932446249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157532024-02-07T19:01:32.723Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.932469286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157542024-02-07T19:01:32.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.932484374Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157552024-02-07T19:01:32.723Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.932498302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157562024-02-07T19:01:32.729Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.932856914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157572024-02-07T19:01:32.729Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933048318Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157582024-02-07T19:01:32.729Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933195398Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157592024-02-07T19:01:32.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933451813Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157602024-02-07T19:01:32.729Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933489068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157612024-02-07T19:01:32.729Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933537502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157622024-02-07T19:01:32.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933571147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157632024-02-07T19:01:32.729Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.93362736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157642024-02-07T19:01:32.729Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933660325Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157652024-02-07T19:01:32.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933781249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157662024-02-07T19:01:32.729Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933817394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157672024-02-07T19:01:32.729Z{"msg":"[0]R flush_numbers[0..12]: [241, 241, 231, 241, 240, 240, 239, 240, 240, 239, 239, 239]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933852019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157682024-02-07T19:01:32.729Z{"msg":"[0]R generation[0..12]: [143, 143, 134, 143, 142, 142, 141, 142, 142, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933874106Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157692024-02-07T19:01:32.729Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933890394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157702024-02-07T19:01:32.729Z{"msg":"[1]R flush_numbers[0..12]: [241, 241, 231, 241, 240, 240, 239, 240, 240, 239, 239, 239]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933911721Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157712024-02-07T19:01:32.730Z{"msg":"[1]R generation[0..12]: [143, 143, 134, 143, 142, 142, 141, 142, 142, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933926159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157722024-02-07T19:01:32.730Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933941377Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157732024-02-07T19:01:32.730ZWait for a query_work_queue command to finish before sending IO
157742024-02-07T19:01:32.730Z{"msg":"[2]R flush_numbers[0..12]: [241, 241, 231, 241, 240, 240, 239, 240, 240, 239, 239, 239]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933958845Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157752024-02-07T19:01:32.730Z{"msg":"[2]R generation[0..12]: [143, 143, 134, 143, 142, 142, 141, 142, 142, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.933986221Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157762024-02-07T19:01:32.730Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934001169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157772024-02-07T19:01:32.730Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934024736Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157782024-02-07T19:01:32.730Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934039194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157792024-02-07T19:01:32.730Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.93406626Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157802024-02-07T19:01:32.730Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934080678Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157812024-02-07T19:01:32.730Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934094286Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157822024-02-07T19:01:32.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934107005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157832024-02-07T19:01:32.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934121093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157842024-02-07T19:01:32.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934146399Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157852024-02-07T19:01:32.730Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934160328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157862024-02-07T19:01:32.730Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934173526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157872024-02-07T19:01:32.730Z{"msg":"8d0a2ae6-baf7-476d-92d7-22c3acd7eafc is now active with session: c4f08d27-8384-487c-a654-2d2822b5aa63","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934186604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157882024-02-07T19:01:32.730Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934205821Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711,"session_id":"c4f08d27-8384-487c-a654-2d2822b5aa63"}
157892024-02-07T19:01:32.730Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:31.934219229Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711}
157902024-02-07T19:01:32.730ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157912024-02-07T19:01:32.730ZLoading write count information from file "/tmp/test_repair_verify.data"
157922024-02-07T19:01:32.730ZRun Repair workload
157932024-02-07T19:01:32.730Z01/30 Write block 221 len 36864 data: 8 8 12 13 11 8 8 9 11
157942024-02-07T19:01:32.730Z02/30 Write block 521 len 12288 data: 9 7 7
157952024-02-07T19:01:32.731Z03/30 Write block 543 len 32768 data: 8 7 7 9 14 15 16 14
157962024-02-07T19:01:32.731Z04/30 Read block 207 len 32768
157972024-02-07T19:01:32.734Z05/30 Write block 290 len 16384 data: 17 16 13 12
157982024-02-07T19:01:32.734Z06/30 Flush
157992024-02-07T19:01:32.734Z07/30 Write block 102 len 32768 data: 8 11 16 18 17 17 16 15
158002024-02-07T19:01:32.734Z08/30 Read block 24 len 40960
158012024-02-07T19:01:32.734Z09/30 Read block 328 len 12288
158022024-02-07T19:01:32.734Z10/30 Read block 414 len 16384
158032024-02-07T19:01:32.737Z11/30 Read block 443 len 32768
158042024-02-07T19:01:32.737Z12/30 Write block 595 len 4096 data: 8
158052024-02-07T19:01:32.738Z13/30 Write block 443 len 12288 data: 13 14 15
158062024-02-07T19:01:32.738Z14/30 Write block 388 len 8192 data: 13 13
158072024-02-07T19:01:32.742Z15/30 Write block 287 len 40960 data: 17 17 16 18 17 14 13 10 11 10
158082024-02-07T19:01:32.742Z16/30 Write block 68 len 28672 data: 13 12 11 12 10 8 6
158092024-02-07T19:01:32.742Z17/30 Flush
158102024-02-07T19:01:32.745Z18/30 Write block 351 len 24576 data: 13 10 9 13 10 7
158112024-02-07T19:01:32.745Z19/30 Read block 23 len 24576
158122024-02-07T19:01:32.745Z20/30 Flush
158132024-02-07T19:01:32.748Z21/30 Write block 560 len 16384 data: 11 9 10 10
158142024-02-07T19:01:32.748Z22/30 Read block 389 len 24576
158152024-02-07T19:01:32.748Z23/30 Read block 358 len 8192
158162024-02-07T19:01:32.748Z24/30 Read block 3 len 12288
158172024-02-07T19:01:32.757Z25/30 Write block 427 len 28672 data: 9 13 12 12 13 11 9
158182024-02-07T19:01:32.757Z26/30 Read block 548 len 40960
158192024-02-07T19:01:32.757Z27/30 Read block 19 len 20480
158202024-02-07T19:01:32.757Z28/30 Read block 15 len 28672
158212024-02-07T19:01:32.757Z29/30 Write block 122 len 40960 data: 14 15 16 16 15 13 15 15 12 9
158222024-02-07T19:01:32.757Z30/30 Write block 418 len 36864 data: 11 8 8 8 6 7 8 13 10
158232024-02-07T19:01:32.757Z----------------------------------------------------------------
158242024-02-07T19:01:32.757Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
158252024-02-07T19:01:32.757ZStates: Active Active Active
158262024-02-07T19:01:32.757ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
158272024-02-07T19:01:32.757Z 1 Acked 1000 Write 9 Done Done Done false
158282024-02-07T19:01:32.757Z 2 Acked 1001 Write 3 Done Sent Done false
158292024-02-07T19:01:32.758Z 3 Acked 1002 Write 8 Done Sent Done false
158302024-02-07T19:01:32.758Z 4 Acked 1003 Read 8 Done Sent Done false
158312024-02-07T19:01:32.758Z 5 Acked 1004 Write 4 Done Sent Done false
158322024-02-07T19:01:32.758Z 6 Acked 1005 Flush 0 Done Sent Done false
158332024-02-07T19:01:32.758Z 7 Acked 1006 Write 8 Done Sent Done false
158342024-02-07T19:01:32.758Z 8 Acked 1007 Read 10 Done Sent Done false
158352024-02-07T19:01:32.758Z 9 Acked 1008 Read 3 Done Sent Done false
158362024-02-07T19:01:32.758Z 10 Acked 1009 Read 4 Done Sent Done false
158372024-02-07T19:01:32.758Z 11 Acked 1010 Read 8 Done Sent Done false
158382024-02-07T19:01:32.758Z 12 Acked 1011 Write 1 Done Sent Done false
158392024-02-07T19:01:32.758Z 13 Acked 1012 Write 3 Done Sent Done false
158402024-02-07T19:01:32.758Z 14 Acked 1013 Write 2 Done Sent Done false
158412024-02-07T19:01:32.758Z 15 Acked 1014 Write 10 Done Sent Done false
158422024-02-07T19:01:32.758Z 16 Acked 1015 Write 7 Done Sent Done false
158432024-02-07T19:01:32.758Z 17 Acked 1016 Flush 0 Done Sent Done false
158442024-02-07T19:01:32.758Z 18 Acked 1017 Write 6 Done Sent Done false
158452024-02-07T19:01:32.758Z 19 Acked 1018 Read 6 Done Sent Done false
158462024-02-07T19:01:32.758Z 20 Acked 1019 Flush 0 Done Sent Done false
158472024-02-07T19:01:32.758Z 21 Acked 1020 Write 4 Done Sent Done false
158482024-02-07T19:01:32.758Z 22 Acked 1021 Read 6 Done Sent Done false
158492024-02-07T19:01:32.758Z 23 Acked 1022 Read 2 Done Sent Done false
158502024-02-07T19:01:32.758Z 24 Acked 1023 Read 3 Done Sent Done false
158512024-02-07T19:01:32.758Z 25 Acked 1024 Write 7 Done Sent Done false
158522024-02-07T19:01:32.758Z 26 Acked 1025 Read 10 Done Sent Done false
158532024-02-07T19:01:32.759Z 27 Acked 1026 Read 5 Done Sent Done false
158542024-02-07T19:01:32.759Z 28 Acked 1027 Read 7 Done Sent Done false
158552024-02-07T19:01:32.759Z 29 Acked 1028 Write 10 Sent Sent Sent false
158562024-02-07T19:01:32.759Z 30 Acked 1029 Write 9 Sent Sent Sent false
158572024-02-07T19:01:32.759Z STATES DS:0 DS:1 DS:2 TOTAL
158582024-02-07T19:01:32.759Z New 0 0 0 0
158592024-02-07T19:01:32.759Z Sent 2 29 2 33
158602024-02-07T19:01:32.759Z Done 28 1 28 57
158612024-02-07T19:01:32.759Z Skipped 0 0 0 0
158622024-02-07T19:01:32.759Z Error 0 0 0 0
158632024-02-07T19:01:32.759ZLast Flush: 1019 0 1019
158642024-02-07T19:01:32.759ZDownstairs last five completed:
158652024-02-07T19:01:32.759ZUpstairs last five completed: 30 29 28 27 26
158662024-02-07T19:01:32.759Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:31.966684889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1711}
158672024-02-07T19:01:32.759ZWrote out file "/tmp/test_repair_verify.data"
158682024-02-07T19:01:32.764Z
158692024-02-07T19:01:32.764ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158702024-02-07T19:01:32.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158712024-02-07T19:01:32.780Z 0 000-019 143 143 143 241 241 241 F F F
158722024-02-07T19:01:32.780Z 1 020-039 143 143 143 241 241 241 F F F
158732024-02-07T19:01:32.780Z 2 040-059 134 134 134 231 231 231 F F F
158742024-02-07T19:01:32.781Z 3 060-079 144 143 144 243 241 243 F F F <---
158752024-02-07T19:01:32.781Z 4 080-099 142 142 142 240 240 240 F F F
158762024-02-07T19:01:32.781Z 5 100-119 144 142 144 243 240 243 F F F <---
158772024-02-07T19:01:32.781Z 6 120-139 141 141 141 239 239 239 T F T <---
158782024-02-07T19:01:32.781Z 7 140-159 142 142 142 240 240 240 F F F
158792024-02-07T19:01:32.781Z 8 160-179 142 142 142 240 240 240 F F F
158802024-02-07T19:01:32.785Z 9 180-199 141 141 141 239 239 239 F F F
158812024-02-07T19:01:32.785Z 10 200-219 141 141 141 239 239 239 F F F
158822024-02-07T19:01:32.785Z 11 220-239 144 141 144 242 239 242 F T F <---
158832024-02-07T19:01:32.785Z 12 240-259 138 138 138 235 235 235 F F F
158842024-02-07T19:01:32.785Z 13 260-279 142 142 142 240 240 240 F F F
158852024-02-07T19:01:32.785Z 14 280-299 144 140 144 243 238 243 F F F <---
158862024-02-07T19:01:32.785Z 15 300-319 139 139 139 237 237 237 F F F
158872024-02-07T19:01:32.785Z 16 320-339 137 137 137 234 234 234 F F F
158882024-02-07T19:01:32.785Z 17 340-359 144 138 144 244 235 244 F F F <---
158892024-02-07T19:01:32.785Z 18 360-379 142 142 142 240 240 240 F F F
158902024-02-07T19:01:32.785Z 19 380-399 144 141 144 243 239 243 F F F <---
158912024-02-07T19:01:32.785Z 20 400-419 136 136 136 233 233 233 T F T <---
158922024-02-07T19:01:32.785Z 21 420-439 140 140 140 238 238 238 T F T <---
158932024-02-07T19:01:32.785Z 22 440-459 144 130 144 243 224 243 F F F <---
158942024-02-07T19:01:32.785Z 23 460-479 141 141 141 239 239 239 F F F
158952024-02-07T19:01:32.785Z 24 480-499 142 142 142 240 240 240 F F F
158962024-02-07T19:01:32.785Z 25 500-519 142 142 142 240 240 240 F F F
158972024-02-07T19:01:32.785Z 26 520-539 144 142 144 242 240 242 F F F <---
158982024-02-07T19:01:32.785Z 27 540-559 144 142 144 242 240 242 F F F <---
158992024-02-07T19:01:32.786Z 28 560-579 137 137 137 234 234 234 T F T <---
159002024-02-07T19:01:32.786Z 29 580-599 144 137 144 243 234 243 F F F <---
159012024-02-07T19:01:32.786ZMax gen: 144, Max flush: 244
159022024-02-07T19:01:32.786ZError: Difference in extent metadata found!
159032024-02-07T19:01:32.786ZA Difference in extent metadata is expected here
159042024-02-07T19:01:32.786ZOn loop 72
159052024-02-07T19:01:32.786Z
159062024-02-07T19:01:32.792ZVerifying data now
159072024-02-07T19:01:38.467ZNo change after verify
159082024-02-07T19:01:38.467ZLoop: 72 Downstairs dump after verify (and repair):
159092024-02-07T19:01:38.482ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159102024-02-07T19:01:38.482Z 0 000-019 143 143 143 241 241 241 F F F
159112024-02-07T19:01:38.482Z 1 020-039 143 143 143 241 241 241 F F F
159122024-02-07T19:01:38.483Z 2 040-059 134 134 134 231 231 231 F F F
159132024-02-07T19:01:38.483Z 3 060-079 144 144 144 243 243 243 F F F
159142024-02-07T19:01:38.483Z 4 080-099 142 142 142 240 240 240 F F F
159152024-02-07T19:01:38.483Z 5 100-119 144 144 144 243 243 243 F F F
159162024-02-07T19:01:38.483Z 6 120-139 145 145 145 245 245 245 F F F
159172024-02-07T19:01:38.483Z 7 140-159 142 142 142 240 240 240 F F F
159182024-02-07T19:01:38.483Z 8 160-179 142 142 142 240 240 240 F F F
159192024-02-07T19:01:38.486Z 9 180-199 141 141 141 239 239 239 F F F
159202024-02-07T19:01:38.487Z 10 200-219 141 141 141 239 239 239 F F F
159212024-02-07T19:01:38.487Z 11 220-239 144 144 144 242 242 242 F F F
159222024-02-07T19:01:38.487Z 12 240-259 138 138 138 235 235 235 F F F
159232024-02-07T19:01:38.487Z 13 260-279 142 142 142 240 240 240 F F F
159242024-02-07T19:01:38.487Z 14 280-299 144 144 144 243 243 243 F F F
159252024-02-07T19:01:38.487Z 15 300-319 139 139 139 237 237 237 F F F
159262024-02-07T19:01:38.487Z 16 320-339 137 137 137 234 234 234 F F F
159272024-02-07T19:01:38.487Z 17 340-359 144 144 144 244 244 244 F F F
159282024-02-07T19:01:38.487Z 18 360-379 142 142 142 240 240 240 F F F
159292024-02-07T19:01:38.487Z 19 380-399 144 144 144 243 243 243 F F F
159302024-02-07T19:01:38.487Z 20 400-419 145 145 145 245 245 245 F F F
159312024-02-07T19:01:38.487Z 21 420-439 145 145 145 245 245 245 F F F
159322024-02-07T19:01:38.487Z 22 440-459 144 144 144 243 243 243 F F F
159332024-02-07T19:01:38.487Z 23 460-479 141 141 141 239 239 239 F F F
159342024-02-07T19:01:38.487Z 24 480-499 142 142 142 240 240 240 F F F
159352024-02-07T19:01:38.487Z 25 500-519 142 142 142 240 240 240 F F F
159362024-02-07T19:01:38.487Z 26 520-539 144 144 144 242 242 242 F F F
159372024-02-07T19:01:38.488Z 27 540-559 144 144 144 242 242 242 F F F
159382024-02-07T19:01:38.488Z 28 560-579 145 145 145 245 245 245 F F F
159392024-02-07T19:01:38.488Z 29 580-599 144 144 144 243 243 243 F F F
159402024-02-07T19:01:38.488ZMax gen: 145, Max flush: 245
159412024-02-07T19:01:38.488Z
159422024-02-07T19:01:38.488ZBegin loop 73
159432024-02-07T19:01:38.488ZDownstairs to restart: 1
159442024-02-07T19:01:38.506Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714101138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719}
159452024-02-07T19:01:38.506Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714389189Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719}
159462024-02-07T19:01:38.506Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714421534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719}
159472024-02-07T19:01:38.506ZCrucible runtime is spawned
159482024-02-07T19:01:38.513Z{"msg":"Crucible 85218b66-a5f6-4a42-9cfb-11a6e1c49546 has session id: f1eb4259-a950-44cb-9483-23cecfcf0d05","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714466138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159492024-02-07T19:01:38.513Z{"msg":"Upstairs opts: Upstairs UUID: 85218b66-a5f6-4a42-9cfb-11a6e1c49546, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71452874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159502024-02-07T19:01:38.513Z{"msg":"Crucible stats registered with UUID: 85218b66-a5f6-4a42-9cfb-11a6e1c49546","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714555156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159512024-02-07T19:01:38.514Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714572584Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719}
159522024-02-07T19:01:38.514Z{"msg":"85218b66-a5f6-4a42-9cfb-11a6e1c49546 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.7146754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159532024-02-07T19:01:38.514Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714722554Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159542024-02-07T19:01:38.514Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714759279Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159552024-02-07T19:01:38.514Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714803833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159562024-02-07T19:01:38.514Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714848576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159572024-02-07T19:01:38.514Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71489454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159582024-02-07T19:01:38.514Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714945443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159592024-02-07T19:01:38.514Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.714979219Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159602024-02-07T19:01:38.514Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.715011665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159612024-02-07T19:01:38.514Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71504185Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159622024-02-07T19:01:38.514Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.715098513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159632024-02-07T19:01:38.514Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.715147906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159642024-02-07T19:01:38.514Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71519191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159652024-02-07T19:01:38.514Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71571245Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159662024-02-07T19:01:38.514Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.715745735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159672024-02-07T19:01:38.514Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.715761463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159682024-02-07T19:01:38.514Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.715776021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159692024-02-07T19:01:38.514Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.716363862Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159702024-02-07T19:01:38.514Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.716406116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159712024-02-07T19:01:38.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71696549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159722024-02-07T19:01:38.514Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717006555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159732024-02-07T19:01:38.514Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717023802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159742024-02-07T19:01:38.515Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717083804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159752024-02-07T19:01:38.515Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717110941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159762024-02-07T19:01:38.515Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717126279Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159772024-02-07T19:01:38.515Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717147506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159782024-02-07T19:01:38.515Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717161544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159792024-02-07T19:01:38.515Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.717680404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159802024-02-07T19:01:38.515Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718126173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159812024-02-07T19:01:38.515Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718175586Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159822024-02-07T19:01:38.515Z{"msg":"[0]R flush_numbers[0..12]: [241, 241, 231, 243, 240, 243, 245, 240, 240, 239, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718205513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159832024-02-07T19:01:38.515Z{"msg":"[0]R generation[0..12]: [143, 143, 134, 144, 142, 144, 145, 142, 142, 141, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718235378Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159842024-02-07T19:01:38.515Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718263035Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159852024-02-07T19:01:38.515ZWait for a query_work_queue command to finish before sending IO
159862024-02-07T19:01:38.515Z{"msg":"[1]R flush_numbers[0..12]: [241, 241, 231, 243, 240, 243, 245, 240, 240, 239, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71829598Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159872024-02-07T19:01:38.515Z{"msg":"[1]R generation[0..12]: [143, 143, 134, 144, 142, 144, 145, 142, 142, 141, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718330146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159882024-02-07T19:01:38.515Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718362551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159892024-02-07T19:01:38.515Z{"msg":"[2]R flush_numbers[0..12]: [241, 241, 231, 243, 240, 243, 245, 240, 240, 239, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718389238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159902024-02-07T19:01:38.515Z{"msg":"[2]R generation[0..12]: [143, 143, 134, 144, 142, 144, 145, 142, 142, 141, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718420744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159912024-02-07T19:01:38.515Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.71844765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159922024-02-07T19:01:38.515Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718476616Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159932024-02-07T19:01:38.515Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718501973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159942024-02-07T19:01:38.515Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718528759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159952024-02-07T19:01:38.515Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718552735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159962024-02-07T19:01:38.515Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718579832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159972024-02-07T19:01:38.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718601939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159982024-02-07T19:01:38.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718629225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
159992024-02-07T19:01:38.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718653112Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
160002024-02-07T19:01:38.516ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
160012024-02-07T19:01:38.516Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718682188Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
160022024-02-07T19:01:38.516Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718705235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
160032024-02-07T19:01:38.516Z{"msg":"85218b66-a5f6-4a42-9cfb-11a6e1c49546 is now active with session: f1eb4259-a950-44cb-9483-23cecfcf0d05","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718732351Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
160042024-02-07T19:01:38.516Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718754228Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719,"session_id":"f1eb4259-a950-44cb-9483-23cecfcf0d05"}
160052024-02-07T19:01:38.516Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:37.718779895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719}
160062024-02-07T19:01:38.516ZLoading write count information from file "/tmp/test_repair_verify.data"
160072024-02-07T19:01:38.516ZRun Repair workload
160082024-02-07T19:01:38.516Z01/30 Write block 482 len 20480 data: 8 11 10 8 8
160092024-02-07T19:01:38.525Z02/30 Write block 360 len 4096 data: 10
160102024-02-07T19:01:38.525Z03/30 Write block 275 len 32768 data: 8 6 7 9 7 8 9 9
160112024-02-07T19:01:38.525Z04/30 Read block 489 len 12288
160122024-02-07T19:01:38.525Z05/30 Write block 563 len 40960 data: 11 13 14 14 15 15 13 13 11 8
160132024-02-07T19:01:38.525Z06/30 Write block 496 len 20480 data: 10 8 10 9 10
160142024-02-07T19:01:38.526Z07/30 Write block 60 len 20480 data: 7 7 7 8 11
160152024-02-07T19:01:38.526Z08/30 Write block 335 len 20480 data: 10 10 10 12 15
160162024-02-07T19:01:38.526Z09/30 Read block 441 len 32768
160172024-02-07T19:01:38.526Z10/30 Read block 307 len 40960
160182024-02-07T19:01:38.526Z11/30 Write block 279 len 12288 data: 8 9 10
160192024-02-07T19:01:38.526Z12/30 Flush
160202024-02-07T19:01:38.526Z13/30 Write block 326 len 8192 data: 7 6
160212024-02-07T19:01:38.526Z14/30 Read block 449 len 40960
160222024-02-07T19:01:38.526Z15/30 Read block 593 len 12288
160232024-02-07T19:01:38.526Z16/30 Read block 509 len 4096
160242024-02-07T19:01:38.533Z17/30 Write block 377 len 40960 data: 10 10 8 9 8 8 11 10 11 11
160252024-02-07T19:01:38.533Z18/30 Write block 6 len 12288 data: 12 15 16
160262024-02-07T19:01:38.533Z19/30 Flush
160272024-02-07T19:01:38.533Z20/30 Write block 136 len 8192 data: 4 5
160282024-02-07T19:01:38.533Z21/30 Write block 450 len 12288 data: 11 12 10
160292024-02-07T19:01:38.533Z22/30 Write block 74 len 28672 data: 7 6 9 10 13 12 10
160302024-02-07T19:01:38.538Z23/30 Write block 87 len 12288 data: 6 7 7
160312024-02-07T19:01:38.538Z24/30 Read block 202 len 40960
160322024-02-07T19:01:38.538Z25/30 Write block 77 len 16384 data: 11 14 13 11
160332024-02-07T19:01:38.538Z26/30 Write block 43 len 20480 data: 8 8 7 7 8
160342024-02-07T19:01:38.538Z27/30 Read block 318 len 20480
160352024-02-07T19:01:38.538Z28/30 Read block 400 len 12288
160362024-02-07T19:01:38.538Z29/30 Read block 47 len 8192
160372024-02-07T19:01:38.538Z30/30 Write block 428 len 24576 data: 14 13 13 14 12 10
160382024-02-07T19:01:38.538Z----------------------------------------------------------------
160392024-02-07T19:01:38.538Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
160402024-02-07T19:01:38.538ZStates: Active Active Active
160412024-02-07T19:01:38.538ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
160422024-02-07T19:01:38.538Z 1 Acked 1000 Write 5 Done Sent Done false
160432024-02-07T19:01:38.538Z 2 Acked 1001 Write 1 Done Sent Done false
160442024-02-07T19:01:38.538Z 3 Acked 1002 Write 8 Done Sent Done false
160452024-02-07T19:01:38.538Z 4 Acked 1003 Read 3 Done Sent Done false
160462024-02-07T19:01:38.538Z 5 Acked 1004 Write 10 Done Sent Done false
160472024-02-07T19:01:38.539Z 6 Acked 1005 Write 5 Done Sent Done false
160482024-02-07T19:01:38.539Z 7 Acked 1006 Write 5 Done Sent Done false
160492024-02-07T19:01:38.539Z 8 Acked 1007 Write 5 Done Sent Done false
160502024-02-07T19:01:38.539Z 9 Acked 1008 Read 8 Done Sent Done false
160512024-02-07T19:01:38.539Z 10 Acked 1009 Read 10 Done Sent Done false
160522024-02-07T19:01:38.539Z 11 Acked 1010 Write 3 Done Sent Done false
160532024-02-07T19:01:38.539Z 12 Acked 1011 Flush 0 Done Sent Done false
160542024-02-07T19:01:38.539Z 13 Acked 1012 Write 2 Done Sent Done false
160552024-02-07T19:01:38.539Z 14 Acked 1013 Read 10 Done Sent Done false
160562024-02-07T19:01:38.539Z 15 Acked 1014 Read 3 Done Sent Done false
160572024-02-07T19:01:38.539Z 16 Acked 1015 Read 1 Done Sent Done false
160582024-02-07T19:01:38.539Z 17 Acked 1016 Write 10 Done Sent Done false
160592024-02-07T19:01:38.539Z 18 Acked 1017 Write 3 Done Sent Done false
160602024-02-07T19:01:38.539Z 19 Acked 1018 Flush 0 Done Sent Done false
160612024-02-07T19:01:38.539Z 20 Acked 1019 Write 2 Done Sent Done false
160622024-02-07T19:01:38.539Z 21 Acked 1020 Write 3 Done Sent Done false
160632024-02-07T19:01:38.539Z 22 Acked 1021 Write 7 Done Sent Done false
160642024-02-07T19:01:38.539Z 23 Acked 1022 Write 3 Done Sent Done false
160652024-02-07T19:01:38.539Z 24 Acked 1023 Read 10 Done Sent Done false
160662024-02-07T19:01:38.539Z 25 Acked 1024 Write 4 Done Sent Done false
160672024-02-07T19:01:38.539Z 26 Acked 1025 Write 5 Done Sent Done false
160682024-02-07T19:01:38.539Z 27 Acked 1026 Read 5 Done Sent Done false
160692024-02-07T19:01:38.539Z 28 Acked 1027 Read 3 Done Sent Done false
160702024-02-07T19:01:38.540Z 29 Acked 1028 Read 2 Done Sent Done false
160712024-02-07T19:01:38.540Z 30 Acked 1029 Write 6 Sent Sent Sent false
160722024-02-07T19:01:38.540Z STATES DS:0 DS:1 DS:2 TOTAL
160732024-02-07T19:01:38.540Z New 0 0 0 0
160742024-02-07T19:01:38.540Z Sent 1 30 1 32
160752024-02-07T19:01:38.540Z Done 29 0 29 58
160762024-02-07T19:01:38.540Z Skipped 0 0 0 0
160772024-02-07T19:01:38.540Z Error 0 0 0 0
160782024-02-07T19:01:38.540ZLast Flush: 1018 0 1018
160792024-02-07T19:01:38.540ZDownstairs last five completed:
160802024-02-07T19:01:38.540ZUpstairs last five completed: 30 29 28 27 26
160812024-02-07T19:01:38.540Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:37.747330443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1719}
160822024-02-07T19:01:38.540ZWrote out file "/tmp/test_repair_verify.data"
160832024-02-07T19:01:38.546Z
160842024-02-07T19:01:38.561ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160852024-02-07T19:01:38.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160862024-02-07T19:01:38.565Z 0 000-019 146 143 146 247 241 247 F F F <---
160872024-02-07T19:01:38.565Z 1 020-039 143 143 143 241 241 241 F F F
160882024-02-07T19:01:38.565Z 2 040-059 134 134 134 231 231 231 T F T <---
160892024-02-07T19:01:38.565Z 3 060-079 146 144 146 246 243 246 T F T <---
160902024-02-07T19:01:38.565Z 4 080-099 142 142 142 240 240 240 T F T <---
160912024-02-07T19:01:38.565Z 5 100-119 144 144 144 243 243 243 F F F
160922024-02-07T19:01:38.565Z 6 120-139 145 145 145 245 245 245 T F T <---
160932024-02-07T19:01:38.565Z 7 140-159 142 142 142 240 240 240 F F F
160942024-02-07T19:01:38.565Z 8 160-179 142 142 142 240 240 240 F F F
160952024-02-07T19:01:38.565Z 9 180-199 141 141 141 239 239 239 F F F
160962024-02-07T19:01:38.565Z 10 200-219 141 141 141 239 239 239 F F F
160972024-02-07T19:01:38.565Z 11 220-239 144 144 144 242 242 242 F F F
160982024-02-07T19:01:38.565Z 12 240-259 138 138 138 235 235 235 F F F
160992024-02-07T19:01:38.565Z 13 260-279 146 142 146 246 240 246 F F F <---
161002024-02-07T19:01:38.565Z 14 280-299 146 144 146 246 243 246 F F F <---
161012024-02-07T19:01:38.565Z 15 300-319 139 139 139 237 237 237 F F F
161022024-02-07T19:01:38.565Z 16 320-339 146 137 146 247 234 247 F F F <---
161032024-02-07T19:01:38.565Z 17 340-359 144 144 144 244 244 244 F F F
161042024-02-07T19:01:38.565Z 18 360-379 146 142 146 247 240 247 F F F <---
161052024-02-07T19:01:38.565Z 19 380-399 146 144 146 247 243 247 F F F <---
161062024-02-07T19:01:38.565Z 20 400-419 145 145 145 245 245 245 F F F
161072024-02-07T19:01:38.566Z 21 420-439 145 145 145 245 245 245 T F T <---
161082024-02-07T19:01:38.566Z 22 440-459 144 144 144 243 243 243 T F T <---
161092024-02-07T19:01:38.566Z 23 460-479 141 141 141 239 239 239 F F F
161102024-02-07T19:01:38.566Z 24 480-499 146 142 146 246 240 246 F F F <---
161112024-02-07T19:01:38.566Z 25 500-519 146 142 146 246 240 246 F F F <---
161122024-02-07T19:01:38.566Z 26 520-539 144 144 144 242 242 242 F F F
161132024-02-07T19:01:38.566Z 27 540-559 144 144 144 242 242 242 F F F
161142024-02-07T19:01:38.566Z 28 560-579 146 145 146 246 245 246 F F F <---
161152024-02-07T19:01:38.566Z 29 580-599 144 144 144 243 243 243 F F F
161162024-02-07T19:01:38.566ZMax gen: 146, Max flush: 247
161172024-02-07T19:01:38.566ZError: Difference in extent metadata found!
161182024-02-07T19:01:38.569ZA Difference in extent metadata is expected here
161192024-02-07T19:01:38.569ZOn loop 73
161202024-02-07T19:01:38.570Z
161212024-02-07T19:01:38.570ZVerifying data now
161222024-02-07T19:01:43.290ZNo change after verify
161232024-02-07T19:01:43.290ZLoop: 73 Downstairs dump after verify (and repair):
161242024-02-07T19:01:43.303ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161252024-02-07T19:01:43.304Z 0 000-019 146 146 146 247 247 247 F F F
161262024-02-07T19:01:43.304Z 1 020-039 143 143 143 241 241 241 F F F
161272024-02-07T19:01:43.304Z 2 040-059 147 147 147 248 248 248 F F F
161282024-02-07T19:01:43.304Z 3 060-079 147 147 147 248 248 248 F F F
161292024-02-07T19:01:43.304Z 4 080-099 147 147 147 248 248 248 F F F
161302024-02-07T19:01:43.304Z 5 100-119 144 144 144 243 243 243 F F F
161312024-02-07T19:01:43.304Z 6 120-139 147 147 147 248 248 248 F F F
161322024-02-07T19:01:43.304Z 7 140-159 142 142 142 240 240 240 F F F
161332024-02-07T19:01:43.304Z 8 160-179 142 142 142 240 240 240 F F F
161342024-02-07T19:01:43.304Z 9 180-199 141 141 141 239 239 239 F F F
161352024-02-07T19:01:43.304Z 10 200-219 141 141 141 239 239 239 F F F
161362024-02-07T19:01:43.304Z 11 220-239 144 144 144 242 242 242 F F F
161372024-02-07T19:01:43.304Z 12 240-259 138 138 138 235 235 235 F F F
161382024-02-07T19:01:43.308Z 13 260-279 146 146 146 246 246 246 F F F
161392024-02-07T19:01:43.308Z 14 280-299 146 146 146 246 246 246 F F F
161402024-02-07T19:01:43.308Z 15 300-319 139 139 139 237 237 237 F F F
161412024-02-07T19:01:43.308Z 16 320-339 146 146 146 247 247 247 F F F
161422024-02-07T19:01:43.308Z 17 340-359 144 144 144 244 244 244 F F F
161432024-02-07T19:01:43.308Z 18 360-379 146 146 146 247 247 247 F F F
161442024-02-07T19:01:43.308Z 19 380-399 146 146 146 247 247 247 F F F
161452024-02-07T19:01:43.308Z 20 400-419 145 145 145 245 245 245 F F F
161462024-02-07T19:01:43.308Z 21 420-439 147 147 147 248 248 248 F F F
161472024-02-07T19:01:43.308Z 22 440-459 147 147 147 248 248 248 F F F
161482024-02-07T19:01:43.308Z 23 460-479 141 141 141 239 239 239 F F F
161492024-02-07T19:01:43.308Z 24 480-499 146 146 146 246 246 246 F F F
161502024-02-07T19:01:43.308Z 25 500-519 146 146 146 246 246 246 F F F
161512024-02-07T19:01:43.308Z 26 520-539 144 144 144 242 242 242 F F F
161522024-02-07T19:01:43.308Z 27 540-559 144 144 144 242 242 242 F F F
161532024-02-07T19:01:43.308Z 28 560-579 146 146 146 246 246 246 F F F
161542024-02-07T19:01:43.308Z 29 580-599 144 144 144 243 243 243 F F F
161552024-02-07T19:01:43.308ZMax gen: 147, Max flush: 248
161562024-02-07T19:01:43.308Z
161572024-02-07T19:01:43.309ZBegin loop 74
161582024-02-07T19:01:43.309ZDownstairs to restart: 1
161592024-02-07T19:01:43.327Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535157198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727}
161602024-02-07T19:01:43.327Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
161612024-02-07T19:01:43.327Z,"time":"2024-02-07T19:01:42.53550582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727}
161622024-02-07T19:01:43.333Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535548915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727}
161632024-02-07T19:01:43.333Z{"msg":"Crucible 559a5bae-869f-4e77-9448-d7eaa14d7ca0 has session id: 647e10c3-9d40-41a6-b379-81b50b5067ac","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535591009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161642024-02-07T19:01:43.334Z{"msg":"Upstairs opts: Upstairs UUID: 559a5bae-869f-4e77-9448-d7eaa14d7ca0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535631783Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161652024-02-07T19:01:43.334Z{"msg":"Crucible stats registered with UUID: 559a5bae-869f-4e77-9448-d7eaa14d7ca0","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535649341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161662024-02-07T19:01:43.334Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535662569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727}
161672024-02-07T19:01:43.334Z{"msg":"559a5bae-869f-4e77-9448-d7eaa14d7ca0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535798781Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161682024-02-07T19:01:43.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535826707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161692024-02-07T19:01:43.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535842645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161702024-02-07T19:01:43.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535857693Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161712024-02-07T19:01:43.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.53588135Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161722024-02-07T19:01:43.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535924764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161732024-02-07T19:01:43.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.53595379Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161742024-02-07T19:01:43.334Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535976647Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161752024-02-07T19:01:43.334Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.535997924Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161762024-02-07T19:01:43.334Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.536016751Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161772024-02-07T19:01:43.334Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.536227773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161782024-02-07T19:01:43.334Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.536304622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161792024-02-07T19:01:43.334Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:42.536423826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161802024-02-07T19:01:43.334Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:42.53647039Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161812024-02-07T19:01:43.334Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:42.536563757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161822024-02-07T19:01:43.335Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.536602712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161832024-02-07T19:01:43.335Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.536649096Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161842024-02-07T19:01:43.335Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.53669611Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161852024-02-07T19:01:43.335Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.537015196Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161862024-02-07T19:01:43.335Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.53706184Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161872024-02-07T19:01:43.335Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.537086797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161882024-02-07T19:01:43.335Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.537109714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161892024-02-07T19:01:43.335Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.537667918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161902024-02-07T19:01:43.335Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.537761215Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161912024-02-07T19:01:43.335Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.538262758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161922024-02-07T19:01:43.335Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.538307621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161932024-02-07T19:01:43.335Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.538346366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161942024-02-07T19:01:43.335Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.538369973Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161952024-02-07T19:01:43.335Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.538384551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161962024-02-07T19:01:43.335Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:42.538403768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161972024-02-07T19:01:44.331Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.538399129Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161982024-02-07T19:01:44.331Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.538521422Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
161992024-02-07T19:01:44.337Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.539395714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162002024-02-07T19:01:44.337Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.539441768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162012024-02-07T19:01:44.337Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.539939521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162022024-02-07T19:01:44.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540545319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162032024-02-07T19:01:44.337Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540591793Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162042024-02-07T19:01:44.337Z{"msg":"[0]R flush_numbers[0..12]: [247, 241, 248, 248, 248, 243, 248, 240, 240, 239, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.54061355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162052024-02-07T19:01:44.337Z{"msg":"[0]R generation[0..12]: [146, 143, 147, 147, 147, 144, 147, 142, 142, 141, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540633357Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162062024-02-07T19:01:44.337Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540662403Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162072024-02-07T19:01:44.338Z{"msg":"[1]R flush_numbers[0..12]: [247, 241, 248, 248, 248, 243, 248, 240, 240, 239, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540681041Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162082024-02-07T19:01:44.338Z{"msg":"[1]R generation[0..12]: [146, 143, 147, 147, 147, 144, 147, 142, 142, 141, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540698488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162092024-02-07T19:01:44.338Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540725404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162102024-02-07T19:01:44.338Z{"msg":"[2]R flush_numbers[0..12]: [247, 241, 248, 248, 248, 243, 248, 240, 240, 239, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540743502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162112024-02-07T19:01:44.338Z{"msg":"[2]R generation[0..12]: [146, 143, 147, 147, 147, 144, 147, 142, 142, 141, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.54076085Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162122024-02-07T19:01:44.338Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540778447Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162132024-02-07T19:01:44.338ZWait for a query_work_queue command to finish before sending IO
162142024-02-07T19:01:44.338Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540808343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162152024-02-07T19:01:44.338Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540826081Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162162024-02-07T19:01:44.338Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540851087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162172024-02-07T19:01:44.338Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540872895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162182024-02-07T19:01:44.338Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540915909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162192024-02-07T19:01:44.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540944425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162202024-02-07T19:01:44.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.540986919Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162212024-02-07T19:01:44.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.541019944Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162222024-02-07T19:01:44.338Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.541063319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162232024-02-07T19:01:44.338Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.541104163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162242024-02-07T19:01:44.338Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
162252024-02-07T19:01:44.338Z"msg":"559a5bae-869f-4e77-9448-d7eaa14d7ca0 is now active with session: 647e10c3-9d40-41a6-b379-81b50b5067ac","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.541137099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162262024-02-07T19:01:44.338Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.541179073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727,"session_id":"647e10c3-9d40-41a6-b379-81b50b5067ac"}
162272024-02-07T19:01:44.338Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:43.54119982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727}
162282024-02-07T19:01:44.338ZLoading write count information from file "/tmp/test_repair_verify.data"
162292024-02-07T19:01:44.338ZRun Repair workload
162302024-02-07T19:01:44.338Z01/30 Flush
162312024-02-07T19:01:44.339Z02/30 Write block 6 len 8192 data: 13 16
162322024-02-07T19:01:44.339Z03/30 Write block 354 len 24576 data: 14 11 8 4 6 7
162332024-02-07T19:01:44.339Z04/30 Read block 494 len 12288
162342024-02-07T19:01:44.344Z05/30 Read block 552 len 4096
162352024-02-07T19:01:44.344Z06/30 Flush
162362024-02-07T19:01:44.344Z07/30 Write block 569 len 20480 data: 14 14 12 9 9
162372024-02-07T19:01:44.344Z08/30 Read block 6 len 12288
162382024-02-07T19:01:44.344Z09/30 Write block 31 len 24576 data: 17 16 15 16 15 15
162392024-02-07T19:01:44.344Z10/30 Write block 576 len 8192 data: 8 8
162402024-02-07T19:01:44.344Z11/30 Read block 513 len 24576
162412024-02-07T19:01:44.344Z12/30 Write block 105 len 12288 data: 19 18 18
162422024-02-07T19:01:44.355Z13/30 Write block 76 len 36864 data: 10 12 15 14 12 9 9 9 8
162432024-02-07T19:01:44.356Z14/30 Write block 590 len 28672 data: 13 15 13 11 10 9 5
162442024-02-07T19:01:44.356Z15/30 Read block 400 len 4096
162452024-02-07T19:01:44.356Z16/30 Read block 22 len 40960
162462024-02-07T19:01:44.356Z17/30 Read block 56 len 4096
162472024-02-07T19:01:44.356Z18/30 Write block 153 len 24576 data: 13 13 10 10 8 11
162482024-02-07T19:01:44.356Z19/30 Read block 205 len 40960
162492024-02-07T19:01:44.356Z20/30 Read block 489 len 16384
162502024-02-07T19:01:44.365Z21/30 Write block 248 len 20480 data: 12 11 10 9 10
162512024-02-07T19:01:44.365Z22/30 Write block 591 len 28672 data: 16 14 12 11 10 6 5
162522024-02-07T19:01:44.365Z23/30 Read block 134 len 8192
162532024-02-07T19:01:44.365Z24/30 Read block 466 len 4096
162542024-02-07T19:01:44.365Z25/30 Write block 561 len 16384 data: 10 11 12 14
162552024-02-07T19:01:44.365Z26/30 Write block 196 len 16384 data: 11 12 14 13
162562024-02-07T19:01:44.365Z27/30 Read block 581 len 20480
162572024-02-07T19:01:44.365Z28/30 Write block 307 len 40960 data: 6 9 10 10 10 9 11 10 13 15
162582024-02-07T19:01:44.366Z29/30 Read block 474 len 16384
162592024-02-07T19:01:44.366Z30/30 Write block 301 len 24576 data: 8 10 6 7 8 7
162602024-02-07T19:01:44.366Z----------------------------------------------------------------
162612024-02-07T19:01:44.366Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:29
162622024-02-07T19:01:44.366ZStates: Active Active Active
162632024-02-07T19:01:44.366ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162642024-02-07T19:01:44.366Z 2 Acked 1001 Write 2 Done Sent Done false
162652024-02-07T19:01:44.367Z 3 Acked 1002 Write 6 Done Sent Done false
162662024-02-07T19:01:44.367Z 4 Acked 1003 Read 3 Done Sent Done false
162672024-02-07T19:01:44.367Z 5 Acked 1004 Read 1 Done Sent Done false
162682024-02-07T19:01:44.367Z 6 Acked 1005 Flush 0 Done Sent Done false
162692024-02-07T19:01:44.367Z 7 Acked 1006 Write 5 Done Sent Done false
162702024-02-07T19:01:44.367Z 8 Acked 1007 Read 3 Done Sent Done false
162712024-02-07T19:01:44.368Z 9 Acked 1008 Write 6 Done Sent Done false
162722024-02-07T19:01:44.368Z 10 Acked 1009 Write 2 Done Sent Done false
162732024-02-07T19:01:44.368Z 11 Acked 1010 Read 6 Done Sent Done false
162742024-02-07T19:01:44.368Z 12 Acked 1011 Write 3 Done Sent Done false
162752024-02-07T19:01:44.368Z 13 Acked 1012 Write 9 Done Sent Done false
162762024-02-07T19:01:44.368Z 14 Acked 1013 Write 7 Done Sent Done false
162772024-02-07T19:01:44.368Z 15 Acked 1014 Read 1 Done Sent Done false
162782024-02-07T19:01:44.369Z 16 Acked 1015 Read 10 Done Sent Done false
162792024-02-07T19:01:44.369Z 17 Acked 1016 Read 1 Done Sent Done false
162802024-02-07T19:01:44.369Z 18 Acked 1017 Write 6 Done Sent Done false
162812024-02-07T19:01:44.369Z 19 Acked 1018 Read 10 Done Sent Done false
162822024-02-07T19:01:44.370Z 20 Acked 1019 Read 4 Done Sent Done false
162832024-02-07T19:01:44.370Z 21 Acked 1020 Write 5 Done Sent Done false
162842024-02-07T19:01:44.370Z 22 Acked 1021 Write 7 Done Sent Done false
162852024-02-07T19:01:44.370Z 23 Acked 1022 Read 2 Done Sent Done false
162862024-02-07T19:01:44.370Z 24 Acked 1023 Read 1 Done Sent Done false
162872024-02-07T19:01:44.371Z 25 Acked 1024 Write 4 Done Sent Done false
162882024-02-07T19:01:44.371Z 26 Acked 1025 Write 4 Done Sent Done false
162892024-02-07T19:01:44.371Z 27 Acked 1026 Read 5 Done Sent Done false
162902024-02-07T19:01:44.371Z 28 Acked 1027 Write 10 Done Sent Done false
162912024-02-07T19:01:44.371Z 29 Acked 1028 Read 4 Done Sent Done false
162922024-02-07T19:01:44.371Z 30 Acked 1029 Write 6 Sent Sent Sent false
162932024-02-07T19:01:44.371Z STATES DS:0 DS:1 DS:2 TOTAL
162942024-02-07T19:01:44.371Z New 0 0 0 0
162952024-02-07T19:01:44.372Z Sent 1 29 1 31
162962024-02-07T19:01:44.372Z Done 28 0 28 56
162972024-02-07T19:01:44.372Z Skipped 0 0 0 0
162982024-02-07T19:01:44.372Z Error 0 0 0 0
162992024-02-07T19:01:44.372ZLast Flush: 1005 1000 1005
163002024-02-07T19:01:44.373ZDownstairs last five completed: 1000
163012024-02-07T19:01:44.373ZUpstairs last five completed: 30 29 28 27 26
163022024-02-07T19:01:44.373Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:43.568283647Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1727}
163032024-02-07T19:01:44.373ZWrote out file "/tmp/test_repair_verify.data"
163042024-02-07T19:01:44.816Z
163052024-02-07T19:01:44.816ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163062024-02-07T19:01:44.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163072024-02-07T19:01:44.816Z 0 000-019 148 146 148 250 247 250 F F F <---
163082024-02-07T19:01:44.816Z 1 020-039 143 143 143 241 241 241 T F T <---
163092024-02-07T19:01:44.816Z 2 040-059 147 147 147 248 248 248 F F F
163102024-02-07T19:01:44.816Z 3 060-079 147 147 147 248 248 248 T F T <---
163112024-02-07T19:01:44.816Z 4 080-099 147 147 147 248 248 248 T F T <---
163122024-02-07T19:01:44.816Z 5 100-119 144 144 144 243 243 243 T F T <---
163132024-02-07T19:01:44.817Z 6 120-139 147 147 147 248 248 248 F F F
163142024-02-07T19:01:44.817Z 7 140-159 142 142 142 240 240 240 T F T <---
163152024-02-07T19:01:44.817Z 8 160-179 142 142 142 240 240 240 F F F
163162024-02-07T19:01:44.817Z 9 180-199 141 141 141 239 239 239 T F T <---
163172024-02-07T19:01:44.817Z 10 200-219 141 141 141 239 239 239 F F F
163182024-02-07T19:01:44.817Z 11 220-239 144 144 144 242 242 242 F F F
163192024-02-07T19:01:44.817Z 12 240-259 138 138 138 235 235 235 T F T <---
163202024-02-07T19:01:44.817Z 13 260-279 146 146 146 246 246 246 F F F
163212024-02-07T19:01:44.817Z 14 280-299 146 146 146 246 246 246 F F F
163222024-02-07T19:01:44.817Z 15 300-319 139 139 139 237 237 237 T F T <---
163232024-02-07T19:01:44.817Z 16 320-339 146 146 146 247 247 247 F F F
163242024-02-07T19:01:44.817Z 17 340-359 148 144 148 250 244 250 F F F <---
163252024-02-07T19:01:44.817Z 18 360-379 146 146 146 247 247 247 F F F
163262024-02-07T19:01:44.818Z 19 380-399 146 146 146 247 247 247 F F F
163272024-02-07T19:01:44.818Z 20 400-419 145 145 145 245 245 245 F F F
163282024-02-07T19:01:44.818Z 21 420-439 147 147 147 248 248 248 F F F
163292024-02-07T19:01:44.818Z 22 440-459 147 147 147 248 248 248 F F F
163302024-02-07T19:01:44.818Z 23 460-479 141 141 141 239 239 239 F F F
163312024-02-07T19:01:44.818Z 24 480-499 146 146 146 246 246 246 F F F
163322024-02-07T19:01:44.818Z 25 500-519 146 146 146 246 246 246 F F F
163332024-02-07T19:01:44.819Z 26 520-539 144 144 144 242 242 242 F F F
163342024-02-07T19:01:44.819Z 27 540-559 144 144 144 242 242 242 F F F
163352024-02-07T19:01:44.819Z 28 560-579 146 146 146 246 246 246 T F T <---
163362024-02-07T19:01:44.819Z 29 580-599 144 144 144 243 243 243 T F T <---
163372024-02-07T19:01:44.819ZMax gen: 148, Max flush: 250
163382024-02-07T19:01:44.819ZError: Difference in extent metadata found!
163392024-02-07T19:01:44.819ZA Difference in extent metadata is expected here
163402024-02-07T19:01:44.819ZOn loop 74
163412024-02-07T19:01:44.819Z
163422024-02-07T19:01:44.820ZVerifying data now
163432024-02-07T19:01:49.008ZNo change after verify
163442024-02-07T19:01:49.009ZLoop: 74 Downstairs dump after verify (and repair):
163452024-02-07T19:01:55.335ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163462024-02-07T19:01:55.335Z 0 000-019 148 148 148 250 250 250 F F F
163472024-02-07T19:01:55.335Z 1 020-039 149 149 149 251 251 251 F F F
163482024-02-07T19:01:55.335Z 2 040-059 147 147 147 248 248 248 F F F
163492024-02-07T19:01:55.336Z 3 060-079 149 149 149 251 251 251 F F F
163502024-02-07T19:01:55.336Z 4 080-099 149 149 149 251 251 251 F F F
163512024-02-07T19:01:55.336Z 5 100-119 149 149 149 251 251 251 F F F
163522024-02-07T19:01:55.336Z 6 120-139 147 147 147 248 248 248 F F F
163532024-02-07T19:01:55.336Z 7 140-159 149 149 149 251 251 251 F F F
163542024-02-07T19:01:55.336Z 8 160-179 142 142 142 240 240 240 F F F
163552024-02-07T19:01:55.336Z 9 180-199 149 149 149 251 251 251 F F F
163562024-02-07T19:01:55.336Z 10 200-219 141 141 141 239 239 239 F F F
163572024-02-07T19:01:55.336Z 11 220-239 144 144 144 242 242 242 F F F
163582024-02-07T19:01:55.336Z 12 240-259 149 149 149 251 251 251 F F F
163592024-02-07T19:01:55.336Z 13 260-279 146 146 146 246 246 246 F F F
163602024-02-07T19:01:55.336Z 14 280-299 146 146 146 246 246 246 F F F
163612024-02-07T19:01:55.336Z 15 300-319 149 149 149 251 251 251 F F F
163622024-02-07T19:01:55.336Z 16 320-339 146 146 146 247 247 247 F F F
163632024-02-07T19:01:55.336Z 17 340-359 148 148 148 250 250 250 F F F
163642024-02-07T19:01:55.336Z 18 360-379 146 146 146 247 247 247 F F F
163652024-02-07T19:01:55.336Z 19 380-399 146 146 146 247 247 247 F F F
163662024-02-07T19:01:55.336Z 20 400-419 145 145 145 245 245 245 F F F
163672024-02-07T19:01:55.336Z 21 420-439 147 147 147 248 248 248 F F F
163682024-02-07T19:01:55.336Z 22 440-459 147 147 147 248 248 248 F F F
163692024-02-07T19:01:55.336Z 23 460-479 141 141 141 239 239 239 F F F
163702024-02-07T19:01:55.336Z 24 480-499 146 146 146 246 246 246 F F F
163712024-02-07T19:01:55.336Z 25 500-519 146 146 146 246 246 246 F F F
163722024-02-07T19:01:55.337Z 26 520-539 144 144 144 242 242 242 F F F
163732024-02-07T19:01:55.337Z 27 540-559 144 144 144 242 242 242 F F F
163742024-02-07T19:01:55.337Z 28 560-579 149 149 149 251 251 251 F F F
163752024-02-07T19:01:55.337Z 29 580-599 149 149 149 251 251 251 F F F
163762024-02-07T19:01:55.337ZMax gen: 149, Max flush: 251
163772024-02-07T19:01:55.337Z
163782024-02-07T19:01:55.337ZBegin loop 75
163792024-02-07T19:01:55.337ZDownstairs to restart: 2
163802024-02-07T19:01:55.337Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.23670332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735}
163812024-02-07T19:01:55.337Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.236979123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735}
163822024-02-07T19:01:55.337Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.236998741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735}
163832024-02-07T19:01:55.337Z{"msg":"Crucible f2470504-a290-4c2a-8e9a-10b92418525c has session id: 6f67eebf-89d1-4053-9a06-aae5ad1fc8b6","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237019388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163842024-02-07T19:01:55.337Z{"msg":"Upstairs opts: Upstairs UUID: f2470504-a290-4c2a-8e9a-10b92418525c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237060172Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163852024-02-07T19:01:55.337Z{"msg":"Crucible stats registered with UUID: f2470504-a290-4c2a-8e9a-10b92418525c","v":Crucible runtime is spawned
163862024-02-07T19:01:55.337Z0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237103596Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163872024-02-07T19:01:55.337Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237172697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735}
163882024-02-07T19:01:55.337Z{"msg":"f2470504-a290-4c2a-8e9a-10b92418525c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237268994Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163892024-02-07T19:01:55.337Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237308508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163902024-02-07T19:01:55.337Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237354062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163912024-02-07T19:01:55.337Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237402056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163922024-02-07T19:01:55.337Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237438971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163932024-02-07T19:01:55.337Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237478156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163942024-02-07T19:01:55.337Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237504802Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163952024-02-07T19:01:55.338Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237556625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163962024-02-07T19:01:55.338Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237608468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163972024-02-07T19:01:55.338Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237704475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163982024-02-07T19:01:55.338Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237760587Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
163992024-02-07T19:01:55.338Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237791833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164002024-02-07T19:01:55.338Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:48.237842066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164012024-02-07T19:01:55.338Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:48.237911737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164022024-02-07T19:01:55.338Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:48.237941673Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164032024-02-07T19:01:55.338Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237955401Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164042024-02-07T19:01:55.338Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.237976018Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164052024-02-07T19:01:55.338Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.238059727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164062024-02-07T19:01:55.338Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.238543032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164072024-02-07T19:01:55.338Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.238582706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164082024-02-07T19:01:55.338Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.238612322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164092024-02-07T19:01:55.338Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.238654957Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164102024-02-07T19:01:55.338Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239203132Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164112024-02-07T19:01:55.338Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239319127Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164122024-02-07T19:01:55.338Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239725602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164132024-02-07T19:01:55.338Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239765176Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164142024-02-07T19:01:55.338Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239784684Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164152024-02-07T19:01:55.339Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239854874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164162024-02-07T19:01:55.339Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.2398855Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164172024-02-07T19:01:55.339Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:48.239900488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164182024-02-07T19:01:55.339Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.239824818Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164192024-02-07T19:01:55.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.239906527Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164202024-02-07T19:01:55.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.240784078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164212024-02-07T19:01:55.339Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.240817943Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164222024-02-07T19:01:55.339Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.241312707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164232024-02-07T19:01:55.339Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.241888959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164242024-02-07T19:01:55.339Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.241938662Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164252024-02-07T19:01:55.339Z{"msg":"[0]R flush_numbers[0..12]: [250, 251, 248, 251, 251, 251, 248, 251, 240, 251, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.241973007Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164262024-02-07T19:01:55.339Z{"msg":"[0]R generation[0..12]: [148, 149, 147, 149, 149, 149, 147, 149, 142, 149, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242013502Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164272024-02-07T19:01:55.339Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242042028Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164282024-02-07T19:01:55.339Z{"msg":"[1]R flush_numbers[0..12]: [250, 251, 248, 251, 251, 251, 248, 251, 240, 251, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242070364Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164292024-02-07T19:01:55.339Z{"msg":"[1]R generation[0..12]: [148, 149, 147, 149, 149, 149, 147, 149, 142, 149, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242089072Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164302024-02-07T19:01:55.339Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242115458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164312024-02-07T19:01:55.339Z{"msg":"[2]R flush_numbers[0..12]: [250, 251, 248, 251, 251, 251, 248, 251, 240, 251, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242133796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
164322024-02-07T19:01:55.339Z,"":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164332024-02-07T19:01:55.339Z{"msg":"[2]R generation[0..12]: [148, 149, 147, 149, 149, 149, 147, 149, 142, 149, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242163952Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164342024-02-07T19:01:55.339Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242181699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164352024-02-07T19:01:55.339Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242206946Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164362024-02-07T19:01:55.339Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242224773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164372024-02-07T19:01:55.339Z{"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242240941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164382024-02-07T19:01:55.340Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242265918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164392024-02-07T19:01:55.340Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242283315Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164402024-02-07T19:01:55.340Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242299353Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164412024-02-07T19:01:55.340Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242316161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164422024-02-07T19:01:55.340Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242341637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164432024-02-07T19:01:55.340Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242359455Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164442024-02-07T19:01:55.340Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242375983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164452024-02-07T19:01:55.387Z{"msg":"f2470504-a290-4c2a-8e9a-10b92418525c is now active with session: 6f67eebf-89d1-4053-9a06-aae5ad1fc8b6","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242392421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164462024-02-07T19:01:55.387Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242415238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735,"session_id":"6f67eebf-89d1-4053-9a06-aae5ad1fc8b6"}
164472024-02-07T19:01:55.387Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:49.242431265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735}
164482024-02-07T19:01:55.387ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
164492024-02-07T19:01:55.387ZLoading write count information from file "/tmp/test_repair_verify.data"
164502024-02-07T19:01:55.387ZRun Repair workload
164512024-02-07T19:01:55.388Z01/30 Write block 94 len 40960 data: 13 15 16 14 12 11 9 8 9 12
164522024-02-07T19:01:55.388Z02/30 Write block 373 len 36864 data: 12 11 11 12 11 11 9 10 9
164532024-02-07T19:01:55.388Z03/30 Read block 50 len 4096
164542024-02-07T19:01:55.388Z04/30 Write block 396 len 12288 data: 10 10 13
164552024-02-07T19:01:55.388Z05/30 Read block 44 len 32768
164562024-02-07T19:01:55.388Z06/30 Write block 582 len 36864 data: 13 11 12 13 12 10 13 12 14
164572024-02-07T19:01:55.388Z07/30 Read block 146 len 20480
164582024-02-07T19:01:55.388Z08/30 Read block 149 len 8192
164592024-02-07T19:01:55.388Z09/30 Read block 357 len 36864
164602024-02-07T19:01:55.388Z10/30 Write block 344 len 24576 data: 11 14 16 16 15 14
164612024-02-07T19:01:55.388Z11/30 Flush
164622024-02-07T19:01:55.388Z12/30 Write block 84 len 4096 data: 9
164632024-02-07T19:01:55.388Z13/30 Read block 293 len 20480
164642024-02-07T19:01:55.388Z14/30 Write block 85 len 24576 data: 7 7 7 8 8 9
164652024-02-07T19:01:55.388Z15/30 Read block 296 len 24576
164662024-02-07T19:01:55.388Z16/30 Read block 174 len 4096
164672024-02-07T19:01:55.388Z17/30 Read block 529 len 32768
164682024-02-07T19:01:55.388Z18/30 Read block 95 len 28672
164692024-02-07T19:01:55.388Z19/30 Flush
164702024-02-07T19:01:55.389Z20/30 Write block 582 len 28672 data: 14 12 13 14 13 11 14
164712024-02-07T19:01:55.389Z21/30 Read block 481 len 24576
164722024-02-07T19:01:55.389Z22/30 Write block 566 len 4096 data: 15
164732024-02-07T19:01:55.389Z23/30 Flush
164742024-02-07T19:01:55.389Z24/30 Write block 112 len 32768 data: 11 10 8 10 13 15 15 17
164752024-02-07T19:01:55.389Z25/30 Read block 358 len 36864
164762024-02-07T19:01:55.389Z26/30 Read block 380 len 12288
164772024-02-07T19:01:55.389Z27/30 Write block 541 len 16384 data: 14 14 9 8
164782024-02-07T19:01:55.389Z28/30 Read block 339 len 24576
164792024-02-07T19:01:55.389Z29/30 Write block 21 len 4096 data: 18
164802024-02-07T19:01:55.389Z30/30 Write block 14 len 32768 data: 13 11 13 16 15 18 19 19
164812024-02-07T19:01:55.389Z----------------------------------------------------------------
164822024-02-07T19:01:55.389Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
164832024-02-07T19:01:55.389ZStates: Active Active Active
164842024-02-07T19:01:55.389ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164852024-02-07T19:01:55.389Z 1 Acked 1000 Write 10 Done Done Sent false
164862024-02-07T19:01:55.389Z 2 Acked 1001 Write 9 Done Done Sent false
164872024-02-07T19:01:55.389Z 3 Acked 1002 Read 1 Done Done Sent false
164882024-02-07T19:01:55.389Z 4 Acked 1003 Write 3 Done Done Sent false
164892024-02-07T19:01:55.389Z 5 Acked 1004 Read 8 Done Done Sent false
164902024-02-07T19:01:55.389Z 6 Acked 1005 Write 9 Done Done Sent false
164912024-02-07T19:01:55.389Z 7 Acked 1006 Read 5 Done Done Sent false
164922024-02-07T19:01:55.389Z 8 Acked 1007 Read 2 Done Done Sent false
164932024-02-07T19:01:55.389Z 9 Acked 1008 Read 9 Done Done Sent false
164942024-02-07T19:01:55.390Z 10 Acked 1009 Write 6 Done Done Sent false
164952024-02-07T19:01:55.390Z 11 Acked 1010 Flush 0 Done Done Sent false
164962024-02-07T19:01:55.390Z 12 Acked 1011 Write 1 Done Done Sent false
164972024-02-07T19:01:55.390Z 13 Acked 1012 Read 5 Done Done Sent false
164982024-02-07T19:01:55.390Z 14 Acked 1013 Write 6 Done Done Sent false
164992024-02-07T19:01:55.390Z 15 Acked 1014 Read 6 Done Done Sent false
165002024-02-07T19:01:55.390Z 16 Acked 1015 Read 1 Done Done Sent false
165012024-02-07T19:01:55.390Z 17 Acked 1016 Read 8 Done Done Sent false
165022024-02-07T19:01:55.390Z 18 Acked 1017 Read 7 Done Done Sent false
165032024-02-07T19:01:55.390Z 19 Acked 1018 Flush 0 Done Done Sent false
165042024-02-07T19:01:55.390Z 20 Acked 1019 Write 7 Done Done Sent false
165052024-02-07T19:01:55.390Z 21 Acked 1020 Read 6 Done Done Sent false
165062024-02-07T19:01:55.390Z 22 Acked 1021 Write 1 Done Done Sent false
165072024-02-07T19:01:55.390Z 23 Acked 1022 Flush 0 Done Done Sent false
165082024-02-07T19:01:55.390Z 24 Acked 1023 Write 8 Done Done Sent false
165092024-02-07T19:01:55.390Z 25 Acked 1024 Read 9 Done Done Sent false
165102024-02-07T19:01:55.390Z 26 Acked 1025 Read 3 Done Done Sent false
165112024-02-07T19:01:55.390Z 27 Acked 1026 Write 4 Done Done Sent false
165122024-02-07T19:01:55.390Z 28 Acked 1027 Read 6 Done Done Sent false
165132024-02-07T19:01:55.390Z 29 Acked 1028 Write 1 Done Done Sent false
165142024-02-07T19:01:55.390Z 30 Acked 1029 Write 8 Sent Sent Sent false
165152024-02-07T19:01:55.390Z STATES DS:0 DS:1 DS:2 TOTAL
165162024-02-07T19:01:55.390Z New 0 0 0 0
165172024-02-07T19:01:55.391Z Sent 1 1 30 32
165182024-02-07T19:01:55.391Z Done 29 29 0 58
165192024-02-07T19:01:55.391Z Skipped 0 0 0 0
165202024-02-07T19:01:55.391Z Error 0 0 0 0
165212024-02-07T19:01:55.391ZLast Flush: 1022 1022 0
165222024-02-07T19:01:55.391ZDownstairs last five completed:
165232024-02-07T19:01:55.391ZUpstairs last five completed: 30 29 28 27 26
165242024-02-07T19:01:55.391Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:49.274553371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1735}
165252024-02-07T19:01:55.391ZWrote out file "/tmp/test_repair_verify.data"
165262024-02-07T19:01:55.391Z
165272024-02-07T19:01:55.391ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
165282024-02-07T19:01:55.391ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165292024-02-07T19:01:55.391Z 0 000-019 148 148 148 250 250 250 T T F <---
165302024-02-07T19:01:55.391Z 1 020-039 149 149 149 251 251 251 T T F <---
165312024-02-07T19:01:55.391Z 2 040-059 147 147 147 248 248 248 F F F
165322024-02-07T19:01:55.391Z 3 060-079 149 149 149 251 251 251 F F F
165332024-02-07T19:01:55.391Z 4 080-099 150 150 149 253 253 251 F F F <---
165342024-02-07T19:01:55.391Z 5 100-119 150 150 149 252 252 251 T T F <---
165352024-02-07T19:01:55.391Z 6 120-139 147 147 147 248 248 248 F F F
165362024-02-07T19:01:55.391Z 7 140-159 149 149 149 251 251 251 F F F
165372024-02-07T19:01:55.391Z 8 160-179 142 142 142 240 240 240 F F F
165382024-02-07T19:01:55.391Z 9 180-199 149 149 149 251 251 251 F F F
165392024-02-07T19:01:55.391Z 10 200-219 141 141 141 239 239 239 F F F
165402024-02-07T19:01:55.392Z 11 220-239 144 144 144 242 242 242 F F F
165412024-02-07T19:01:55.392Z 12 240-259 149 149 149 251 251 251 F F F
165422024-02-07T19:01:55.392Z 13 260-279 146 146 146 246 246 246 F F F
165432024-02-07T19:01:55.392Z 14 280-299 146 146 146 246 246 246 F F F
165442024-02-07T19:01:55.392Z 15 300-319 149 149 149 251 251 251 F F F
165452024-02-07T19:01:55.406Z 16 320-339 146 146 146 247 247 247 F F F
165462024-02-07T19:01:55.406Z 17 340-359 150 150 148 252 252 250 F F F <---
165472024-02-07T19:01:55.406Z 18 360-379 150 150 146 252 252 247 F F F <---
165482024-02-07T19:01:55.406Z 19 380-399 150 150 146 252 252 247 F F F <---
165492024-02-07T19:01:55.406Z 20 400-419 145 145 145 245 245 245 F F F
165502024-02-07T19:01:55.407Z 21 420-439 147 147 147 248 248 248 F F F
165512024-02-07T19:01:55.407Z 22 440-459 147 147 147 248 248 248 F F F
165522024-02-07T19:01:55.407Z 23 460-479 141 141 141 239 239 239 F F F
165532024-02-07T19:01:55.407Z 24 480-499 146 146 146 246 246 246 F F F
165542024-02-07T19:01:55.407Z 25 500-519 146 146 146 246 246 246 F F F
165552024-02-07T19:01:55.407Z 26 520-539 144 144 144 242 242 242 F F F
165562024-02-07T19:01:55.407Z 27 540-559 144 144 144 242 242 242 T T F <---
165572024-02-07T19:01:55.407Z 28 560-579 150 150 149 254 254 251 F F F <---
165582024-02-07T19:01:55.407Z 29 580-599 150 150 149 254 254 251 F F F <---
165592024-02-07T19:01:55.407ZMax gen: 150, Max flush: 254
165602024-02-07T19:01:55.407ZError: Difference in extent metadata found!
165612024-02-07T19:01:55.407ZA Difference in extent metadata is expected here
165622024-02-07T19:01:55.407ZOn loop 75
165632024-02-07T19:01:55.407Z
165642024-02-07T19:01:55.407ZVerifying data now
165652024-02-07T19:01:55.407ZNo change after verify
165662024-02-07T19:01:55.407ZLoop: 75 Downstairs dump after verify (and repair):
165672024-02-07T19:01:55.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165682024-02-07T19:01:55.408Z 0 000-019 151 151 151 255 255 255 F F F
165692024-02-07T19:01:55.408Z 1 020-039 151 151 151 255 255 255 F F F
165702024-02-07T19:01:55.408Z 2 040-059 147 147 147 248 248 248 F F F
165712024-02-07T19:01:55.408Z 3 060-079 149 149 149 251 251 251 F F F
165722024-02-07T19:01:55.408Z 4 080-099 150 150 150 253 253 253 F F F
165732024-02-07T19:01:55.408Z 5 100-119 151 151 151 255 255 255 F F F
165742024-02-07T19:01:55.408Z 6 120-139 147 147 147 248 248 248 F F F
165752024-02-07T19:01:55.408Z 7 140-159 149 149 149 251 251 251 F F F
165762024-02-07T19:01:55.408Z 8 160-179 142 142 142 240 240 240 F F F
165772024-02-07T19:01:55.408Z 9 180-199 149 149 149 251 251 251 F F F
165782024-02-07T19:01:55.408Z 10 200-219 141 141 141 239 239 239 F F F
165792024-02-07T19:01:55.408Z 11 220-239 144 144 144 242 242 242 F F F
165802024-02-07T19:01:55.408Z 12 240-259 149 149 149 251 251 251 F F F
165812024-02-07T19:01:55.408Z 13 260-279 146 146 146 246 246 246 F F F
165822024-02-07T19:01:55.408Z 14 280-299 146 146 146 246 246 246 F F F
165832024-02-07T19:01:55.408Z 15 300-319 149 149 149 251 251 251 F F F
165842024-02-07T19:01:55.408Z 16 320-339 146 146 146 247 247 247 F F F
165852024-02-07T19:01:55.408Z 17 340-359 150 150 150 252 252 252 F F F
165862024-02-07T19:01:55.409Z 18 360-379 150 150 150 252 252 252 F F F
165872024-02-07T19:01:55.409Z 19 380-399 150 150 150 252 252 252 F F F
165882024-02-07T19:01:55.409Z 20 400-419 145 145 145 245 245 245 F F F
165892024-02-07T19:01:55.409Z 21 420-439 147 147 147 248 248 248 F F F
165902024-02-07T19:01:55.409Z 22 440-459 147 147 147 248 248 248 F F F
165912024-02-07T19:01:55.409Z 23 460-479 141 141 141 239 239 239 F F F
165922024-02-07T19:01:55.409Z 24 480-499 146 146 146 246 246 246 F F F
165932024-02-07T19:01:55.409Z 25 500-519 146 146 146 246 246 246 F F F
165942024-02-07T19:01:55.409Z 26 520-539 144 144 144 242 242 242 F F F
165952024-02-07T19:01:55.409Z 27 540-559 151 151 151 255 255 255 F F F
165962024-02-07T19:01:55.409Z 28 560-579 150 150 150 254 254 254 F F F
165972024-02-07T19:01:55.409Z 29 580-599 150 150 150 254 254 254 F F F
165982024-02-07T19:01:55.409ZMax gen: 151, Max flush: 255
165992024-02-07T19:01:55.409Z
166002024-02-07T19:01:55.409ZBegin loop 76
166012024-02-07T19:01:55.409ZDownstairs to restart: 1
166022024-02-07T19:01:55.409Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859083511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743}
166032024-02-07T19:01:55.409Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
166042024-02-07T19:01:55.409Z2024-02-07T19:01:53.859371382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743}
166052024-02-07T19:01:55.409Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859414526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743}
166062024-02-07T19:01:55.409Z{"msg":"Crucible 0912b255-06ca-445e-9d2d-4eaf87b55a56 has session id: 989f5f65-2c27-481a-9804-e7d5db046435","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.85945567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166072024-02-07T19:01:55.409Z{"msg":"Upstairs opts: Upstairs UUID: 0912b255-06ca-445e-9d2d-4eaf87b55a56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859512663Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166082024-02-07T19:01:55.409Z{"msg":"Crucible stats registered with UUID: 0912b255-06ca-445e-9d2d-4eaf87b55a56","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859558007Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166092024-02-07T19:01:55.410Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859600261Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743}
166102024-02-07T19:01:55.410Z{"msg":"0912b255-06ca-445e-9d2d-4eaf87b55a56 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859705987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166112024-02-07T19:01:55.410Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859732993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166122024-02-07T19:01:55.410Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859749621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166132024-02-07T19:01:55.410Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859783506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166142024-02-07T19:01:55.410Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859817481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166152024-02-07T19:01:55.410Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859841198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166162024-02-07T19:01:55.410Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859873124Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166172024-02-07T19:01:55.410Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859888872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166182024-02-07T19:01:55.410Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859961792Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166192024-02-07T19:01:55.410Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.859993928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166202024-02-07T19:01:55.410Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860129879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166212024-02-07T19:01:55.410Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860188511Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166222024-02-07T19:01:55.410Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860225826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166232024-02-07T19:01:55.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860827735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166242024-02-07T19:01:55.410Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860870419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166252024-02-07T19:01:55.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860917633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166262024-02-07T19:01:55.410Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.860948099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166272024-02-07T19:01:55.410Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.861476747Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166282024-02-07T19:01:55.410Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.861607799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166292024-02-07T19:01:55.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.861753879Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166302024-02-07T19:01:55.410Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.861807912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166312024-02-07T19:01:55.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862042851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166322024-02-07T19:01:55.410Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862096003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166332024-02-07T19:01:55.411Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862131699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166342024-02-07T19:01:55.411Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862169154Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166352024-02-07T19:01:55.411Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862207298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166362024-02-07T19:01:55.411Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862224676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166372024-02-07T19:01:55.411Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862429668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166382024-02-07T19:01:55.411Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862886146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166392024-02-07T19:01:55.411Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862938799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166402024-02-07T19:01:55.411Z{"msg":"[0]R flush_numbers[0..12]: [255, 255, 248, 251, 253, 255, 248, 251, 240, 251, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862958947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166412024-02-07T19:01:55.411Z{"msg":"[0]R generation[0..12]: [151, 151, 147, 149, 150, 151, 147, 149, 142, 149, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862974025Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166422024-02-07T19:01:55.411Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.862988772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166432024-02-07T19:01:55.411Z{"msg":"[1]R flush_numbers[0..12]: [255, 255, 248, 251, 253, 255, 248, 251, 240, 251, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863003081Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166442024-02-07T19:01:55.411Z{"msg":"[1]R generation[0..12]: [151, 151, 147, 149, 150, 151, 147, 149, 142, 149, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863017829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166452024-02-07T19:01:55.444Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863032737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166462024-02-07T19:01:55.444Z{"msg":"[2]R flush_numbers[0..12]: [255, 255, 248, 251, 253, 255, 248, 251, 240, 251, 239, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863056134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166472024-02-07T19:01:55.444Z{"msg":"[2]R generation[0..12]: [151, 151, 147, 149, 150, 151, 147, 149, 142, 149, 141, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863070712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166482024-02-07T19:01:55.444ZWait for a query_work_queue command to finish before sending IO
166492024-02-07T19:01:55.444Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863096108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166502024-02-07T19:01:55.445Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863110396Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166512024-02-07T19:01:55.445Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863130903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166522024-02-07T19:01:55.445Z{"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863144302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166532024-02-07T19:01:55.445Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863164259Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166542024-02-07T19:01:55.446Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863178487Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166552024-02-07T19:01:55.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863197344Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166562024-02-07T19:01:55.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863211033Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166572024-02-07T19:01:55.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.86323051Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166582024-02-07T19:01:55.446Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863244108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166592024-02-07T19:01:55.447Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863262866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166602024-02-07T19:01:55.447Z{"msg":"0912b255-06ca-445e-9d2d-4eaf87b55a56 is now active with session: 989f5f65-2c27-481a-9804-e7d5db046435","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863276134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166612024-02-07T19:01:55.447Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863295231Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743,"session_id":"989f5f65-2c27-481a-9804-e7d5db046435"}
166622024-02-07T19:01:55.447Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:53.863307769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743}
166632024-02-07T19:01:55.447ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
166642024-02-07T19:01:55.447ZLoading write count information from file "/tmp/test_repair_verify.data"
166652024-02-07T19:01:55.447ZRun Repair workload
166662024-02-07T19:01:55.447ZError: Difference in extent metadata found!
166672024-02-07T19:01:55.447Z01/30 Write block 268 len 36864 data: 11 10 7 8 8 9 7 9 7
166682024-02-07T19:01:55.447Z02/30 Flush
166692024-02-07T19:01:55.447Z03/30 Write block 161 len 40960 data: 11 12 14 17 16 13 12 16 15 14
166702024-02-07T19:01:55.447Z04/30 Read block 557 len 16384
166712024-02-07T19:01:55.447Z05/30 Read block 343 len 28672
166722024-02-07T19:01:55.448Z06/30 Read block 393 len 36864
166732024-02-07T19:01:55.448Z07/30 Flush
166742024-02-07T19:01:55.448Z08/30 Write block 536 len 20480 data: 14 15 12 13 13
166752024-02-07T19:01:55.448Z09/30 Read block 268 len 4096
166762024-02-07T19:01:55.448Z10/30 Write block 434 len 8192 data: 7 8
166772024-02-07T19:01:55.448Z11/30 Write block 207 len 4096 data: 9
166782024-02-07T19:01:55.448Z12/30 Flush
166792024-02-07T19:01:55.448Z13/30 Write block 279 len 36864 data: 9 10 11 10 8 14 14 15 18
166802024-02-07T19:01:55.448Z14/30 Read block 0 len 8192
166812024-02-07T19:01:55.448Z15/30 Write block 474 len 40960 data: 9 10 11 10 9 10 10 9 9 12
166822024-02-07T19:01:55.448Z16/30 Read block 88 len 32768
166832024-02-07T19:01:55.448Z17/30 Write block 469 len 16384 data: 6 8 8 8
166842024-02-07T19:01:55.449Z18/30 Read block 201 len 36864
166852024-02-07T19:01:55.449Z19/30 Flush
166862024-02-07T19:01:55.449Z20/30 Write block 115 len 4096 data: 11
166872024-02-07T19:01:55.449Z21/30 Write block 515 len 8192 data: 15 14
166882024-02-07T19:01:55.449Z22/30 Write block 509 len 36864 data: 13 15 12 16 13 16 16 15 12
166892024-02-07T19:01:55.449Z23/30 Write block 538 len 4096 data: 13
166902024-02-07T19:01:55.449Z24/30 Flush
166912024-02-07T19:01:55.449Z25/30 Write block 26 len 36864 data: 13 13 14 18 17 18 17 16 17
166922024-02-07T19:01:55.449Z26/30 Read block 566 len 8192
166932024-02-07T19:01:55.449Z27/30 Read block 107 len 12288
166942024-02-07T19:01:55.449Z28/30 Read block 174 len 32768
166952024-02-07T19:01:55.449Z29/30 Write block 180 len 36864 data: 11 6 6 5 5 7 7 9 11
166962024-02-07T19:01:55.449Z30/30 Write block 492 len 28672 data: 11 10 13 7 11 9 11
166972024-02-07T19:01:55.450Z----------------------------------------------------------------
166982024-02-07T19:01:55.450Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:28
166992024-02-07T19:01:55.450ZStates: Active Active Active
167002024-02-07T19:01:55.450ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167012024-02-07T19:01:55.450Z 3 Acked 1002 Write 10 Done Done Done false
167022024-02-07T19:01:55.450Z 4 Acked 1003 Read 4 Done Sent Done false
167032024-02-07T19:01:55.450Z 5 Acked 1004 Read 7 Done Sent Done false
167042024-02-07T19:01:55.450Z 6 Acked 1005 Read 9 Done Sent Done false
167052024-02-07T19:01:55.450Z 7 Acked 1006 Flush 0 Done Sent Done false
167062024-02-07T19:01:55.450Z 8 Acked 1007 Write 5 Done Sent Done false
167072024-02-07T19:01:55.450Z 9 Acked 1008 Read 1 Done Sent Done false
167082024-02-07T19:01:55.450Z 10 Acked 1009 Write 2 Done Sent Done false
167092024-02-07T19:01:55.451Z 11 Acked 1010 Write 1 Done Sent Done false
167102024-02-07T19:01:55.451Z 12 Acked 1011 Flush 0 Done Sent Done false
167112024-02-07T19:01:55.451Z 13 Acked 1012 Write 9 Done Sent Done false
167122024-02-07T19:01:55.451Z 14 Acked 1013 Read 2 Done Sent Done false
167132024-02-07T19:01:55.451Z 15 Acked 1014 Write 10 Done Sent Done false
167142024-02-07T19:01:55.451Z 16 Acked 1015 Read 8 Done Sent Done false
167152024-02-07T19:01:55.451Z 17 Acked 1016 Write 4 Done Sent Done false
167162024-02-07T19:01:55.451Z 18 Acked 1017 Read 9 Done Sent Done false
167172024-02-07T19:01:55.451Z 19 Acked 1018 Flush 0 Done Sent Done false
167182024-02-07T19:01:55.451Z 20 Acked 1019 Write 1 Done Sent Done false
167192024-02-07T19:01:55.451Z 21 Acked 1020 Write 2 Done Sent Done false
167202024-02-07T19:01:55.451Z 22 Acked 1021 Write 9 Done Sent Done false
167212024-02-07T19:01:55.451Z 23 Acked 1022 Write 1 Done Sent Done false
167222024-02-07T19:01:55.452Z 24 Acked 1023 Flush 0 Done Sent Done false
167232024-02-07T19:01:55.452Z 25 Acked 1024 Write 9 Done Sent Done false
167242024-02-07T19:01:55.452Z 26 Acked 1025 Read 2 Done Sent Done false
167252024-02-07T19:01:55.452Z 27 Acked 1026 Read 3 Done Sent Done false
167262024-02-07T19:01:55.452Z 28 Acked 1027 Read 8 Done Sent Done false
167272024-02-07T19:01:55.452Z 29 Acked 1028 Write 9 Done Sent Done false
167282024-02-07T19:01:55.452Z 30 Acked 1029 Write 7 Sent Sent Sent false
167292024-02-07T19:01:55.452Z STATES DS:0 DS:1 DS:2 TOTAL
167302024-02-07T19:01:55.452Z New 0 0 0 0
167312024-02-07T19:01:55.452Z Sent 1 27 1 29
167322024-02-07T19:01:55.452Z Done 27 1 27 55
167332024-02-07T19:01:55.452Z Skipped 0 0 0 0
167342024-02-07T19:01:55.452Z Error 0 0 0 0
167352024-02-07T19:01:55.453ZLast Flush: 1023 1001 1023
167362024-02-07T19:01:55.453ZDownstairs last five completed: 1001 1000
167372024-02-07T19:01:55.453ZUpstairs last five completed: 30 29 28 27 26
167382024-02-07T19:01:55.453Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:53.897655164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1743}
167392024-02-07T19:01:55.453ZWrote out file "/tmp/test_repair_verify.data"
167402024-02-07T19:01:55.453Z
167412024-02-07T19:01:55.453ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167422024-02-07T19:01:55.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167432024-02-07T19:01:55.453Z 0 000-019 151 151 151 255 255 255 F F F
167442024-02-07T19:01:55.453Z 1 020-039 151 151 151 255 255 255 T F T <---
167452024-02-07T19:01:55.459Z 2 040-059 147 147 147 248 248 248 F F F
167462024-02-07T19:01:55.459Z 3 060-079 149 149 149 251 251 251 F F F
167472024-02-07T19:01:55.459Z 4 080-099 150 150 150 253 253 253 F F F
167482024-02-07T19:01:55.459Z 5 100-119 152 151 152 260 255 260 F F F <---
167492024-02-07T19:01:55.459Z 6 120-139 147 147 147 248 248 248 F F F
167502024-02-07T19:01:55.459Z 7 140-159 149 149 149 251 251 251 F F F
167512024-02-07T19:01:55.459Z 8 160-179 152 142 152 257 240 257 F T F <---
167522024-02-07T19:01:55.459Z 9 180-199 149 149 149 251 251 251 T F T <---
167532024-02-07T19:01:55.459Z 10 200-219 152 141 152 258 239 258 F F F <---
167542024-02-07T19:01:55.459Z 11 220-239 144 144 144 242 242 242 F F F
167552024-02-07T19:01:55.460Z 12 240-259 149 149 149 251 251 251 F F F
167562024-02-07T19:01:55.460Z 13 260-279 152 152 152 259 256 259 F F F <---
167572024-02-07T19:01:55.460Z 14 280-299 152 146 152 259 246 259 F F F <---
167582024-02-07T19:01:55.460Z 15 300-319 149 149 149 251 251 251 F F F
167592024-02-07T19:01:55.460Z 16 320-339 146 146 146 247 247 247 F F F
167602024-02-07T19:01:55.460Z 17 340-359 150 150 150 252 252 252 F F F
167612024-02-07T19:01:55.460Z 18 360-379 150 150 150 252 252 252 F F F
167622024-02-07T19:01:55.460Z 19 380-399 150 150 150 252 252 252 F F F
167632024-02-07T19:01:55.460Z 20 400-419 145 145 145 245 245 245 F F F
167642024-02-07T19:01:55.460Z 21 420-439 152 147 152 258 248 258 F F F <---
167652024-02-07T19:01:55.460Z 22 440-459 147 147 147 248 248 248 F F F
167662024-02-07T19:01:55.460Z 23 460-479 152 141 152 259 239 259 F F F <---
167672024-02-07T19:01:55.460Z 24 480-499 152 146 152 259 246 259 T F T <---
167682024-02-07T19:01:55.461Z 25 500-519 152 146 152 260 246 260 F F F <---
167692024-02-07T19:01:55.461Z 26 520-539 152 144 152 260 242 260 F F F <---
167702024-02-07T19:01:55.461Z 27 540-559 152 151 152 258 255 258 F F F <---
167712024-02-07T19:01:55.461Z 28 560-579 150 150 150 254 254 254 F F F
167722024-02-07T19:01:55.461Z 29 580-599 150 150 150 254 254 254 F F F
167732024-02-07T19:01:55.461ZMax gen: 152, Max flush: 260
167742024-02-07T19:01:55.461ZA Difference in extent metadata is expected here
167752024-02-07T19:01:55.461ZOn loop 76
167762024-02-07T19:01:55.461Z
167772024-02-07T19:01:55.461ZVerifying data now
167782024-02-07T19:02:00.435ZNo change after verify
167792024-02-07T19:02:00.435ZLoop: 76 Downstairs dump after verify (and repair):
167802024-02-07T19:02:00.442ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167812024-02-07T19:02:00.442Z 0 000-019 151 151 151 255 255 255 F F F
167822024-02-07T19:02:00.442Z 1 020-039 153 153 153 261 261 261 F F F
167832024-02-07T19:02:00.442Z 2 040-059 147 147 147 248 248 248 F F F
167842024-02-07T19:02:00.442Z 3 060-079 149 149 149 251 251 251 F F F
167852024-02-07T19:02:00.442Z 4 080-099 150 150 150 253 253 253 F F F
167862024-02-07T19:02:00.442Z 5 100-119 152 152 152 260 260 260 F F F
167872024-02-07T19:02:00.442Z 6 120-139 147 147 147 248 248 248 F F F
167882024-02-07T19:02:00.443Z 7 140-159 149 149 149 251 251 251 F F F
167892024-02-07T19:02:00.443Z 8 160-179 152 152 152 257 257 257 F F F
167902024-02-07T19:02:00.443Z 9 180-199 153 153 153 261 261 261 F F F
167912024-02-07T19:02:00.443Z 10 200-219 152 152 152 258 258 258 F F F
167922024-02-07T19:02:00.443Z 11 220-239 144 144 144 242 242 242 F F F
167932024-02-07T19:02:00.443Z 12 240-259 149 149 149 251 251 251 F F F
167942024-02-07T19:02:00.443Z 13 260-279 152 152 152 259 259 259 F F F
167952024-02-07T19:02:00.443Z 14 280-299 152 152 152 259 259 259 F F F
167962024-02-07T19:02:00.443Z 15 300-319 149 149 149 251 251 251 F F F
167972024-02-07T19:02:00.443Z 16 320-339 146 146 146 247 247 247 F F F
167982024-02-07T19:02:00.443Z 17 340-359 150 150 150 252 252 252 F F F
167992024-02-07T19:02:00.443Z 18 360-379 150 150 150 252 252 252 F F F
168002024-02-07T19:02:00.443Z 19 380-399 150 150 150 252 252 252 F F F
168012024-02-07T19:02:00.443Z 20 400-419 145 145 145 245 245 245 F F F
168022024-02-07T19:02:00.443Z 21 420-439 152 152 152 258 258 258 F F F
168032024-02-07T19:02:00.443Z 22 440-459 147 147 147 248 248 248 F F F
168042024-02-07T19:02:00.443Z 23 460-479 152 152 152 259 259 259 F F F
168052024-02-07T19:02:00.443Z 24 480-499 153 153 153 261 261 261 F F F
168062024-02-07T19:02:00.443Z 25 500-519 152 152 152 260 260 260 F F F
168072024-02-07T19:02:00.443Z 26 520-539 152 152 152 260 260 260 F F F
168082024-02-07T19:02:00.443Z 27 540-559 152 152 152 258 258 258 F F F
168092024-02-07T19:02:00.443Z 28 560-579 150 150 150 254 254 254 F F F
168102024-02-07T19:02:00.443Z 29 580-599 150 150 150 254 254 254 F F F
168112024-02-07T19:02:00.443ZMax gen: 153, Max flush: 261
168122024-02-07T19:02:00.444Z
168132024-02-07T19:02:00.444ZBegin loop 77
168142024-02-07T19:02:00.444ZDownstairs to restart: 1
168152024-02-07T19:02:00.444Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592330016Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751}
168162024-02-07T19:02:00.444Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592633095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751}
168172024-02-07T19:02:00.444Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592664131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751}
168182024-02-07T19:02:00.444Z{"msg":"Crucible fd70996a-85c1-4884-aeda-fb66f388af5c has session id: da69e0c7-4a9c-4d7c-b44e-cc0d8277c193","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592704045Z","Crucible runtime is spawned
168192024-02-07T19:02:00.444Zhostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168202024-02-07T19:02:00.444Z{"msg":"Upstairs opts: Upstairs UUID: fd70996a-85c1-4884-aeda-fb66f388af5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592874922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168212024-02-07T19:02:00.444Z{"msg":"Crucible stats registered with UUID: fd70996a-85c1-4884-aeda-fb66f388af5c","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592923116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168222024-02-07T19:02:00.444Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592951632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751}
168232024-02-07T19:02:00.444Z{"msg":"fd70996a-85c1-4884-aeda-fb66f388af5c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.59296666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168242024-02-07T19:02:00.444Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.592991077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168252024-02-07T19:02:00.444Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593024202Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168262024-02-07T19:02:00.444Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593055578Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168272024-02-07T19:02:00.444Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593073645Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168282024-02-07T19:02:00.444Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593091163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168292024-02-07T19:02:00.444Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593108481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168302024-02-07T19:02:00.444Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593133337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168312024-02-07T19:02:00.444Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593163013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168322024-02-07T19:02:00.444Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593207337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168332024-02-07T19:02:00.444Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:59.59326301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168342024-02-07T19:02:00.444Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:59.593314733Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168352024-02-07T19:02:00.444Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593369365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168362024-02-07T19:02:00.445Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593428717Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168372024-02-07T19:02:00.445Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:01:59.593456534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168382024-02-07T19:02:00.445Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.59348463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168392024-02-07T19:02:00.445Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593513066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168402024-02-07T19:02:00.445Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.593663735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168412024-02-07T19:02:00.445Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.594171987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168422024-02-07T19:02:00.445Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.594243347Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168432024-02-07T19:02:00.445Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.59429418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168442024-02-07T19:02:00.445Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.594323726Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168452024-02-07T19:02:00.445Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.594745009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168462024-02-07T19:02:00.445Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.594893419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168472024-02-07T19:02:00.445Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.595214426Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168482024-02-07T19:02:00.445Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.595261849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168492024-02-07T19:02:00.445Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.595287696Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168502024-02-07T19:02:00.445Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.595431476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168512024-02-07T19:02:00.445Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.595460603Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168522024-02-07T19:02:00.445Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:01:59.59547735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168532024-02-07T19:02:01.492Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.595024101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168542024-02-07T19:02:01.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.5951077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168552024-02-07T19:02:01.506Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.595980922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168562024-02-07T19:02:01.506Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.596032275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168572024-02-07T19:02:01.506Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.596535197Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168582024-02-07T19:02:01.506Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597146194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168592024-02-07T19:02:01.506Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597185949Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168602024-02-07T19:02:01.506Z{"msg":"[0]R flush_numbers[0..12]: [255, 261, 248, 251, 253, 260, 248, 251, 257, 261, 258, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597203886Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168612024-02-07T19:02:01.506Z{"msg":"[0]R generation[0..12]: [151, 153, 147, 149, 150, 152, 147, 149, 152, 153, 152, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597226563Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168622024-02-07T19:02:01.506Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597242551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168632024-02-07T19:02:01.506Z{"msg":"[1]R flush_numbers[0..12]: [255, 261, 248, 251, 253, 260, 248, 251, 257, 261, 258, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597264158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168642024-02-07T19:02:01.506Z{"msg":"[1]R generation[0..12]: [151, 153, 147, 149, 150, 152, 147, 149, 152, 153, 152, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597279516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168652024-02-07T19:02:01.506Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597301983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168662024-02-07T19:02:01.506Z{"msg":"[2]R flush_numbers[0..12]: [255, 261, 248, 251, 253, 260, 248, 251, 257, 261, 258, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597317201Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168672024-02-07T19:02:01.506Z{"msg":"[2]R generation[0..12]: [151, 153, 147, 149, 150, 152, 147, 149, 152, 153, 152, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597331889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168682024-02-07T19:02:01.506ZWait for a query_work_queue command to finish before sending IO
168692024-02-07T19:02:01.506Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597360985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168702024-02-07T19:02:01.506Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597375803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168712024-02-07T19:02:01.506Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597389881Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168722024-02-07T19:02:01.507Z{"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597409578Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168732024-02-07T19:02:01.507Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597424606Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168742024-02-07T19:02:01.507Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597451653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168752024-02-07T19:02:01.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597465241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168762024-02-07T19:02:01.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597478939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168772024-02-07T19:02:01.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597499076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168782024-02-07T19:02:01.507Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597512915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168792024-02-07T19:02:01.507Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597526013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168802024-02-07T19:02:01.507Z{"msg":"fd70996a-85c1-4884-aeda-fb66f388af5c is now active with session: da69e0c7-4a9c-4d7c-b44e-cc0d8277c193","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597539941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168812024-02-07T19:02:01.507Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597553429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751,"session_id":"da69e0c7-4a9c-4d7c-b44e-cc0d8277c193"}
168822024-02-07T19:02:01.507Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:00.597565847Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751}
168832024-02-07T19:02:01.507ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168842024-02-07T19:02:01.507ZLoading write count information from file "/tmp/test_repair_verify.data"
168852024-02-07T19:02:01.507ZRun Repair workload
168862024-02-07T19:02:01.507Z01/30 Write block 169 len 16384 data: 16 15 13 12
168872024-02-07T19:02:01.507Z02/30 Read block 182 len 12288
168882024-02-07T19:02:01.507Z03/30 Write block 373 len 24576 data: 13 12 12 13 12 12
168892024-02-07T19:02:01.507Z04/30 Write block 294 len 24576 data: 11 12 11 12 11 8
168902024-02-07T19:02:01.507Z05/30 Read block 73 len 8192
168912024-02-07T19:02:01.507Z06/30 Read block 327 len 40960
168922024-02-07T19:02:01.507Z07/30 Read block 127 len 32768
168932024-02-07T19:02:01.507Z08/30 Read block 487 len 12288
168942024-02-07T19:02:01.507Z09/30 Flush
168952024-02-07T19:02:01.507Z10/30 Write block 368 len 8192 data: 13 14
168962024-02-07T19:02:01.508Z11/30 Read block 317 len 8192
168972024-02-07T19:02:01.508Z12/30 Write block 276 len 28672 data: 8 8 10 10 11 12 11
168982024-02-07T19:02:01.508Z13/30 Read block 176 len 28672
168992024-02-07T19:02:01.508Z14/30 Read block 80 len 20480
169002024-02-07T19:02:01.508Z15/30 Read block 30 len 32768
169012024-02-07T19:02:01.508Z16/30 Read block 57 len 16384
169022024-02-07T19:02:01.508Z17/30 Read block 132 len 28672
169032024-02-07T19:02:01.508Z18/30 Write block 562 len 24576 data: 12 13 15 15 16 16
169042024-02-07T19:02:01.508Z19/30 Read block 335 len 40960
169052024-02-07T19:02:01.508Z20/30 Read block 367 len 16384
169062024-02-07T19:02:01.508Z21/30 Read block 23 len 12288
169072024-02-07T19:02:01.508Z22/30 Write block 513 len 36864 data: 14 17 17 16 13 14 12 12 10
169082024-02-07T19:02:01.508Z23/30 Write block 560 len 32768 data: 12 11 13 14 16 16 17 17
169092024-02-07T19:02:01.508Z24/30 Write block 65 len 24576 data: 11 14 13 14 13 12
169102024-02-07T19:02:01.508Z25/30 Write block 63 len 8192 data: 9 12
169112024-02-07T19:02:01.508Z26/30 Write block 143 len 32768 data: 12 12 12 14 15 16 19 13
169122024-02-07T19:02:01.508Z27/30 Write block 214 len 28672 data: 9 7 6 6 7 7 8
169132024-02-07T19:02:01.508Z28/30 Write block 254 len 4096 data: 11
169142024-02-07T19:02:01.508Z29/30 Read block 369 len 40960
169152024-02-07T19:02:01.508Z30/30 Write block 129 len 32768 data: 16 13 10 9 9 9 7 5
169162024-02-07T19:02:01.508Z----------------------------------------------------------------
169172024-02-07T19:02:01.508Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
169182024-02-07T19:02:01.508ZStates: Active Active Active
169192024-02-07T19:02:01.508ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169202024-02-07T19:02:01.509Z 1 Acked 1000 Write 4 Done Done Done false
169212024-02-07T19:02:01.509Z 2 Acked 1001 Read 3 Done Done Done false
169222024-02-07T19:02:01.509Z 3 Acked 1002 Write 6 Done Sent Done false
169232024-02-07T19:02:01.509Z 4 Acked 1003 Write 6 Done Sent Done false
169242024-02-07T19:02:01.509Z 5 Acked 1004 Read 2 Done Sent Done false
169252024-02-07T19:02:01.509Z 6 Acked 1005 Read 10 Done Sent Done false
169262024-02-07T19:02:01.509Z 7 Acked 1006 Read 8 Done Sent Done false
169272024-02-07T19:02:01.509Z 8 Acked 1007 Read 3 Done Sent Done false
169282024-02-07T19:02:01.509Z 9 Acked 1008 Flush 0 Done Sent Done false
169292024-02-07T19:02:01.509Z 10 Acked 1009 Write 2 Done Sent Done false
169302024-02-07T19:02:01.509Z 11 Acked 1010 Read 2 Done Sent Done false
169312024-02-07T19:02:01.509Z 12 Acked 1011 Write 7 Done Sent Done false
169322024-02-07T19:02:01.509Z 13 Acked 1012 Read 7 Done Sent Done false
169332024-02-07T19:02:01.509Z 14 Acked 1013 Read 5 Done Sent Done false
169342024-02-07T19:02:01.509Z 15 Acked 1014 Read 8 Done Sent Done false
169352024-02-07T19:02:01.509Z 16 Acked 1015 Read 4 Done Sent Done false
169362024-02-07T19:02:01.509Z 17 Acked 1016 Read 7 Done Sent Done false
169372024-02-07T19:02:01.509Z 18 Acked 1017 Write 6 Done Sent Done false
169382024-02-07T19:02:01.509Z 19 Acked 1018 Read 10 Done Sent Done false
169392024-02-07T19:02:01.509Z 20 Acked 1019 Read 4 Done Sent Done false
169402024-02-07T19:02:01.509Z 21 Acked 1020 Read 3 Done Sent Done false
169412024-02-07T19:02:01.509Z 22 Acked 1021 Write 9 Done Sent Done false
169422024-02-07T19:02:01.509Z 23 Acked 1022 Write 8 Done Sent Done false
169432024-02-07T19:02:01.509Z 24 Acked 1023 Write 6 Done Sent Done false
169442024-02-07T19:02:01.510Z 25 Acked 1024 Write 2 Done Sent Done false
169452024-02-07T19:02:01.510Z 26 Acked 1025 Write 8 Done Sent Done false
169462024-02-07T19:02:01.510Z 27 Acked 1026 Write 7 Done Sent Done false
169472024-02-07T19:02:01.510Z 28 Acked 1027 Write 1 Done Sent Done false
169482024-02-07T19:02:01.510Z 29 Acked 1028 Read 10 Done Sent Done false
169492024-02-07T19:02:01.510Z 30 Acked 1029 Write 8 Sent Sent Sent false
169502024-02-07T19:02:01.510Z STATES DS:0 DS:1 DS:2 TOTAL
169512024-02-07T19:02:01.510Z New 0 0 0 0
169522024-02-07T19:02:01.510Z Sent 1 28 1 30
169532024-02-07T19:02:01.510Z Done 29 2 29 60
169542024-02-07T19:02:01.510Z Skipped 0 0 0 0
169552024-02-07T19:02:01.518Z Error 0 0 0 0
169562024-02-07T19:02:01.518ZLast Flush: 1008 0 1008
169572024-02-07T19:02:01.518ZDownstairs last five completed:
169582024-02-07T19:02:01.518ZUpstairs last five completed: 30 29 28 27 26
169592024-02-07T19:02:01.518Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:00.627444026Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1751}
169602024-02-07T19:02:01.518ZWrote out file "/tmp/test_repair_verify.data"
169612024-02-07T19:02:01.518Z
169622024-02-07T19:02:01.518ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
169632024-02-07T19:02:01.518ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169642024-02-07T19:02:01.518Z 0 000-019 151 151 151 255 255 255 F F F
169652024-02-07T19:02:01.518Z 1 020-039 153 153 153 261 261 261 F F F
169662024-02-07T19:02:01.518Z 2 040-059 147 147 147 248 248 248 F F F
169672024-02-07T19:02:01.518Z 3 060-079 149 149 149 251 251 251 T F T <---
169682024-02-07T19:02:01.518Z 4 080-099 150 150 150 253 253 253 F F F
169692024-02-07T19:02:01.518Z 5 100-119 152 152 152 260 260 260 F F F
169702024-02-07T19:02:01.519Z 6 120-139 147 147 147 248 248 248 T F T <---
169712024-02-07T19:02:01.519Z 7 140-159 149 149 149 251 251 251 T F T <---
169722024-02-07T19:02:01.519Z 8 160-179 154 152 154 262 257 262 F T F <---
169732024-02-07T19:02:01.519Z 9 180-199 153 153 153 261 261 261 F F F
169742024-02-07T19:02:01.519Z 10 200-219 152 152 152 258 258 258 T F T <---
169752024-02-07T19:02:01.519Z 11 220-239 144 144 144 242 242 242 T F T <---
169762024-02-07T19:02:01.519Z 12 240-259 149 149 149 251 251 251 T F T <---
169772024-02-07T19:02:01.519Z 13 260-279 152 152 152 259 259 259 T F T <---
169782024-02-07T19:02:01.520Z 14 280-299 154 152 154 262 259 262 T F T <---
169792024-02-07T19:02:01.520Z 15 300-319 149 149 149 251 251 251 F F F
169802024-02-07T19:02:01.520Z 16 320-339 146 146 146 247 247 247 F F F
169812024-02-07T19:02:01.520Z 17 340-359 150 150 150 252 252 252 F F F
169822024-02-07T19:02:01.520Z 18 360-379 154 150 154 262 252 262 T F T <---
169832024-02-07T19:02:01.520Z 19 380-399 150 150 150 252 252 252 F F F
169842024-02-07T19:02:01.520Z 20 400-419 145 145 145 245 245 245 F F F
169852024-02-07T19:02:01.520Z 21 420-439 152 152 152 258 258 258 F F F
169862024-02-07T19:02:01.520Z 22 440-459 147 147 147 248 248 248 F F F
169872024-02-07T19:02:01.520Z 23 460-479 152 152 152 259 259 259 F F F
169882024-02-07T19:02:01.520Z 24 480-499 153 153 153 261 261 261 F F F
169892024-02-07T19:02:01.521Z 25 500-519 152 152 152 260 260 260 T F T <---
169902024-02-07T19:02:01.521Z 26 520-539 152 152 152 260 260 260 T F T <---
169912024-02-07T19:02:01.521Z 27 540-559 152 152 152 258 258 258 F F F
169922024-02-07T19:02:01.521Z 28 560-579 150 150 150 254 254 254 T F T <---
169932024-02-07T19:02:01.521Z 29 580-599 150 150 150 254 254 254 F F F
169942024-02-07T19:02:01.521ZMax gen: 154, Max flush: 262
169952024-02-07T19:02:01.521ZError: Difference in extent metadata found!
169962024-02-07T19:02:01.521ZA Difference in extent metadata is expected here
169972024-02-07T19:02:01.522ZOn loop 77
169982024-02-07T19:02:01.522Z
169992024-02-07T19:02:01.522ZVerifying data now
170002024-02-07T19:02:06.101ZNo change after verify
170012024-02-07T19:02:06.101ZLoop: 77 Downstairs dump after verify (and repair):
170022024-02-07T19:02:06.116ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170032024-02-07T19:02:06.116Z 0 000-019 151 151 151 255 255 255 F F F
170042024-02-07T19:02:06.116Z 1 020-039 153 153 153 261 261 261 F F F
170052024-02-07T19:02:06.116Z 2 040-059 147 147 147 248 248 248 F F F
170062024-02-07T19:02:06.116Z 3 060-079 155 155 155 263 263 263 F F F
170072024-02-07T19:02:06.116Z 4 080-099 150 150 150 253 253 253 F F F
170082024-02-07T19:02:06.117Z 5 100-119 152 152 152 260 260 260 F F F
170092024-02-07T19:02:06.117Z 6 120-139 155 155 155 263 263 263 F F F
170102024-02-07T19:02:06.117Z 7 140-159 155 155 155 263 263 263 F F F
170112024-02-07T19:02:06.117Z 8 160-179 154 154 154 262 262 262 F F F
170122024-02-07T19:02:06.117Z 9 180-199 153 153 153 261 261 261 F F F
170132024-02-07T19:02:06.117Z 10 200-219 155 155 155 263 263 263 F F F
170142024-02-07T19:02:06.120Z 11 220-239 155 155 155 263 263 263 F F F
170152024-02-07T19:02:06.120Z 12 240-259 155 155 155 263 263 263 F F F
170162024-02-07T19:02:06.121Z 13 260-279 155 155 155 263 263 263 F F F
170172024-02-07T19:02:06.121Z 14 280-299 155 155 155 263 263 263 F F F
170182024-02-07T19:02:06.121Z 15 300-319 149 149 149 251 251 251 F F F
170192024-02-07T19:02:06.121Z 16 320-339 146 146 146 247 247 247 F F F
170202024-02-07T19:02:06.121Z 17 340-359 150 150 150 252 252 252 F F F
170212024-02-07T19:02:06.121Z 18 360-379 155 155 155 263 263 263 F F F
170222024-02-07T19:02:06.121Z 19 380-399 150 150 150 252 252 252 F F F
170232024-02-07T19:02:06.121Z 20 400-419 145 145 145 245 245 245 F F F
170242024-02-07T19:02:06.121Z 21 420-439 152 152 152 258 258 258 F F F
170252024-02-07T19:02:06.121Z 22 440-459 147 147 147 248 248 248 F F F
170262024-02-07T19:02:06.121Z 23 460-479 152 152 152 259 259 259 F F F
170272024-02-07T19:02:06.121Z 24 480-499 153 153 153 261 261 261 F F F
170282024-02-07T19:02:06.121Z 25 500-519 155 155 155 263 263 263 F F F
170292024-02-07T19:02:06.121Z 26 520-539 155 155 155 263 263 263 F F F
170302024-02-07T19:02:06.121Z 27 540-559 152 152 152 258 258 258 F F F
170312024-02-07T19:02:06.121Z 28 560-579 155 155 155 263 263 263 F F F
170322024-02-07T19:02:06.121Z 29 580-599 150 150 150 254 254 254 F F F
170332024-02-07T19:02:06.121ZMax gen: 155, Max flush: 263
170342024-02-07T19:02:06.121Z
170352024-02-07T19:02:06.121ZBegin loop 78
170362024-02-07T19:02:06.121ZDownstairs to restart: 2
170372024-02-07T19:02:06.138Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.343904742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759}
170382024-02-07T19:02:06.138Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344132921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759}
170392024-02-07T19:02:06.138Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344150609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759}
170402024-02-07T19:02:06.138Z{"msg":"Crucible 817b260a-fa55-4312-9216-a3bb7bcd8aad has session id: 247cd682-9aad-45e3-8c29-704b317d8822","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344172495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170412024-02-07T19:02:06.144Z{"msg":"Upstairs opts: Upstairs UUID: 817b260a-fa55-4312-9216-a3bb7bcd8aad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344202222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170422024-02-07T19:02:06.144Z{"msg":"Crucible stats registered with UUID: 817b260a-fa55-4312-9216-a3bb7bcd8aad","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344228438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170432024-02-07T19:02:06.144ZCrucible runtime is spawned
170442024-02-07T19:02:06.144Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344350621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759}
170452024-02-07T19:02:06.144Z{"msg":"817b260a-fa55-4312-9216-a3bb7bcd8aad active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344430001Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170462024-02-07T19:02:06.144Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344456667Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170472024-02-07T19:02:06.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344473225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170482024-02-07T19:02:06.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344502931Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170492024-02-07T19:02:06.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344519859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170502024-02-07T19:02:06.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344553744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170512024-02-07T19:02:06.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.34458166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170522024-02-07T19:02:06.145Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344642902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170532024-02-07T19:02:06.145Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344682246Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170542024-02-07T19:02:06.145Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344708053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170552024-02-07T19:02:06.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344789402Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170562024-02-07T19:02:06.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344838585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170572024-02-07T19:02:06.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.344873981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170582024-02-07T19:02:06.145Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.34554327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170592024-02-07T19:02:06.145Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.345580095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170602024-02-07T19:02:06.145Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.345596893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170612024-02-07T19:02:06.145Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.345646066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170622024-02-07T19:02:06.145Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346166446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170632024-02-07T19:02:06.145Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346312366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170642024-02-07T19:02:06.145Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346698064Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170652024-02-07T19:02:06.145Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346741628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170662024-02-07T19:02:06.145Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346768255Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170672024-02-07T19:02:06.145Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346840425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170682024-02-07T19:02:06.145Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.3468762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170692024-02-07T19:02:06.145Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346894297Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170702024-02-07T19:02:06.146Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346923463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170712024-02-07T19:02:06.146Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.346953159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170722024-02-07T19:02:06.146Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.347462941Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170732024-02-07T19:02:06.146Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.347934027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170742024-02-07T19:02:06.146Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.347971222Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170752024-02-07T19:02:06.146Z{"msg":"[0]R flush_numbers[0..12]: [255, 261, 248, 263, 253, 260, 263, 263, 262, 261, 263, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.347987839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170762024-02-07T19:02:06.146Z{"msg":"[0]R generation[0..12]: [151, 153, 147, 155, 150, 152, 155, 155, 154, 153, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348010196Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170772024-02-07T19:02:06.146Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348025464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170782024-02-07T19:02:06.146Z{"msg":"[1]R flush_numbers[0..12]: [255, 261, 248, 263, 253, 260, 263, 263, 262, 261, 263, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348046822Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170792024-02-07T19:02:06.151Z{"msg":"[1]R generation[0..12]: [151, 153, 147, 155, 150, 152, 155, 155, 154, 153, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348061249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170802024-02-07T19:02:06.151Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348087476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170812024-02-07T19:02:06.151Z{"msg":"[2]R flush_numbers[0..12]: [255, 261, 248, 263, 253, 260, 263, 263, 262, 261, 263, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348102324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170822024-02-07T19:02:06.151Z{"msg":"[2]R generation[0..12]: [151, 153, 147, 155, 150, 152, 155, 155, 154, 153, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348116512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170832024-02-07T19:02:06.151Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348138589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
170842024-02-07T19:02:06.151Z247cd682-9aad-45e3-8c29-704b317d8822"}
170852024-02-07T19:02:06.151Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348155687Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170862024-02-07T19:02:06.151Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348177154Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170872024-02-07T19:02:06.151Z{"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348190952Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170882024-02-07T19:02:06.151Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.34820447Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170892024-02-07T19:02:06.152Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348231846Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170902024-02-07T19:02:06.152Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348246104Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170912024-02-07T19:02:06.152Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348267052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170922024-02-07T19:02:06.152Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.34828107Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170932024-02-07T19:02:06.152Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348301367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170942024-02-07T19:02:06.152Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348314745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170952024-02-07T19:02:06.152Z{"msg":"817b260a-fa55-4312-9216-a3bb7bcd8aad is now active with session: 247cd682-9aad-45e3-8c29-704b317d8822","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348334133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170962024-02-07T19:02:06.152Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.34835359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759,"session_id":"247cd682-9aad-45e3-8c29-704b317d8822"}
170972024-02-07T19:02:06.152Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:05.348373147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759}
170982024-02-07T19:02:06.152ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170992024-02-07T19:02:06.152ZLoading write count information from file "/tmp/test_repair_verify.data"
171002024-02-07T19:02:06.152ZRun Repair workload
171012024-02-07T19:02:06.152Z01/30 Write block 330 len 28672 data: 10 12 11 12 12 11 11
171022024-02-07T19:02:06.152Z02/30 Read block 202 len 32768
171032024-02-07T19:02:06.152Z03/30 Read block 175 len 8192
171042024-02-07T19:02:06.152Z04/30 Read block 316 len 12288
171052024-02-07T19:02:06.152Z05/30 Write block 494 len 12288 data: 14 8 12
171062024-02-07T19:02:06.152Z06/30 Read block 23 len 24576
171072024-02-07T19:02:06.152Z07/30 Read block 145 len 20480
171082024-02-07T19:02:06.157Z08/30 Write block 23 len 12288 data: 19 16 15
171092024-02-07T19:02:06.157Z09/30 Flush
171102024-02-07T19:02:06.158Z10/30 Flush
171112024-02-07T19:02:06.158Z11/30 Write block 420 len 36864 data: 9 9 7 8 9 14 11 10 15
171122024-02-07T19:02:06.158Z12/30 Read block 516 len 12288
171132024-02-07T19:02:06.158Z13/30 Flush
171142024-02-07T19:02:06.161Z14/30 Write block 325 len 12288 data: 8 8 7
171152024-02-07T19:02:06.161Z15/30 Write block 570 len 16384 data: 15 13 10 10
171162024-02-07T19:02:06.161Z16/30 Write block 359 len 40960 data: 8 11 12 14 13 10 9 13 13 14
171172024-02-07T19:02:06.161Z17/30 Write block 397 len 4096 data: 11
171182024-02-07T19:02:06.161Z18/30 Write block 150 len 36864 data: 14 13 11 14 14 11 11 9 12
171192024-02-07T19:02:06.161Z19/30 Read block 3 len 12288
171202024-02-07T19:02:06.168Z20/30 Read block 320 len 24576
171212024-02-07T19:02:06.168Z21/30 Read block 304 len 4096
171222024-02-07T19:02:06.168Z22/30 Flush
171232024-02-07T19:02:06.172Z23/30 Write block 208 len 24576 data: 12 10 9 10 10 11
171242024-02-07T19:02:06.172Z24/30 Read block 381 len 24576
171252024-02-07T19:02:06.172Z25/30 Read block 92 len 32768
171262024-02-07T19:02:06.172Z26/30 Read block 109 len 24576
171272024-02-07T19:02:06.172Z27/30 Write block 499 len 40960 data: 10 11 9 13 15 13 16 14 12 13
171282024-02-07T19:02:06.172Z28/30 Read block 143 len 40960
171292024-02-07T19:02:06.176Z29/30 Read block 122 len 32768
171302024-02-07T19:02:06.176Z30/30 Read block 389 len 4096
171312024-02-07T19:02:06.176Z----------------------------------------------------------------
171322024-02-07T19:02:06.176Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
171332024-02-07T19:02:06.176ZStates: Active Active Active
171342024-02-07T19:02:06.176ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
171352024-02-07T19:02:06.176Z 1 Acked 1000 Write 7 Done Done Sent false
171362024-02-07T19:02:06.176Z 2 Acked 1001 Read 8 Done Done Sent false
171372024-02-07T19:02:06.176Z 3 Acked 1002 Read 2 Done Done Sent false
171382024-02-07T19:02:06.176Z 4 Acked 1003 Read 3 Done Done Sent false
171392024-02-07T19:02:06.176Z 5 Acked 1004 Write 3 Done Done Sent false
171402024-02-07T19:02:06.176Z 6 Acked 1005 Read 6 Done Done Sent false
171412024-02-07T19:02:06.176Z 7 Acked 1006 Read 5 Done Done Sent false
171422024-02-07T19:02:06.177Z 8 Acked 1007 Write 3 Done Done Sent false
171432024-02-07T19:02:06.177Z 9 Acked 1008 Flush 0 Done Done Sent false
171442024-02-07T19:02:06.177Z 10 Acked 1009 Flush 0 Done Done Sent false
171452024-02-07T19:02:06.177Z 11 Acked 1010 Write 9 Done Done Sent false
171462024-02-07T19:02:06.177Z 12 Acked 1011 Read 3 Done Done Sent false
171472024-02-07T19:02:06.177Z 13 Acked 1012 Flush 0 Done Done Sent false
171482024-02-07T19:02:06.177Z 14 Acked 1013 Write 3 Done Done Sent false
171492024-02-07T19:02:06.177Z 15 Acked 1014 Write 4 Done Done Sent false
171502024-02-07T19:02:06.177Z 16 Acked 1015 Write 10 Done Done Sent false
171512024-02-07T19:02:06.177Z 17 Acked 1016 Write 1 Done Done Sent false
171522024-02-07T19:02:06.177Z 18 Acked 1017 Write 9 Done Done Sent false
171532024-02-07T19:02:06.177Z 19 Acked 1018 Read 3 Done Done Sent false
171542024-02-07T19:02:06.177Z 20 Acked 1019 Read 6 Done Done Sent false
171552024-02-07T19:02:06.177Z 21 Acked 1020 Read 1 Done Done Sent false
171562024-02-07T19:02:06.177Z 22 Acked 1021 Flush 0 Done Done Sent false
171572024-02-07T19:02:06.177Z 23 Acked 1022 Write 6 Done Done Sent false
171582024-02-07T19:02:06.177Z 24 Acked 1023 Read 6 Done Done Sent false
171592024-02-07T19:02:06.177Z 25 Acked 1024 Read 8 Done Done Sent false
171602024-02-07T19:02:06.177Z 26 Acked 1025 Read 6 Done Done Sent false
171612024-02-07T19:02:06.177Z 27 Acked 1026 Write 10 Done Done Sent false
171622024-02-07T19:02:06.177Z 28 Acked 1027 Read 10 Done Done Sent false
171632024-02-07T19:02:06.177Z 29 Acked 1028 Read 8 Done Done Sent false
171642024-02-07T19:02:06.177Z 30 Acked 1029 Read 1 Done Done Sent false
171652024-02-07T19:02:06.177Z STATES DS:0 DS:1 DS:2 TOTAL
171662024-02-07T19:02:06.178Z New 0 0 0 0
171672024-02-07T19:02:06.178Z Sent 0 0 30 30
171682024-02-07T19:02:06.178Z Done 30 30 0 60
171692024-02-07T19:02:06.178Z Skipped 0 0 0 0
171702024-02-07T19:02:06.178Z Error 0 0 0 0
171712024-02-07T19:02:06.178ZLast Flush: 1021 1021 0
171722024-02-07T19:02:06.178ZDownstairs last five completed:
171732024-02-07T19:02:06.178ZUpstairs last five completed: 30 29 28 27 26
171742024-02-07T19:02:06.178Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:05.380014158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1759}
171752024-02-07T19:02:06.178ZWrote out file "/tmp/test_repair_verify.data"
171762024-02-07T19:02:06.182Z
171772024-02-07T19:02:06.182ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171782024-02-07T19:02:06.198ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171792024-02-07T19:02:06.198Z 0 000-019 151 151 151 255 255 255 F F F
171802024-02-07T19:02:06.198Z 1 020-039 156 156 153 264 264 261 F F F <---
171812024-02-07T19:02:06.198Z 2 040-059 147 147 147 248 248 248 F F F
171822024-02-07T19:02:06.198Z 3 060-079 155 155 155 263 263 263 F F F
171832024-02-07T19:02:06.198Z 4 080-099 150 150 150 253 253 253 F F F
171842024-02-07T19:02:06.198Z 5 100-119 152 152 152 260 260 260 F F F
171852024-02-07T19:02:06.198Z 6 120-139 155 155 155 263 263 263 F F F
171862024-02-07T19:02:06.198Z 7 140-159 156 156 155 267 267 263 F F F <---
171872024-02-07T19:02:06.198Z 8 160-179 154 154 154 262 262 262 F F F
171882024-02-07T19:02:06.198Z 9 180-199 153 153 153 261 261 261 F F F
171892024-02-07T19:02:06.204Z 10 200-219 155 155 155 263 263 263 T T F <---
171902024-02-07T19:02:06.204Z 11 220-239 155 155 155 263 263 263 F F F
171912024-02-07T19:02:06.204Z 12 240-259 155 155 155 263 263 263 F F F
171922024-02-07T19:02:06.204Z 13 260-279 155 155 155 263 263 263 F F F
171932024-02-07T19:02:06.204Z 14 280-299 155 155 155 263 263 263 F F F
171942024-02-07T19:02:06.204Z 15 300-319 149 149 149 251 251 251 F F F
171952024-02-07T19:02:06.204Z 16 320-339 156 156 146 267 267 247 F F F <---
171962024-02-07T19:02:06.204Z 17 340-359 156 156 150 267 267 252 F F F <---
171972024-02-07T19:02:06.204Z 18 360-379 156 156 155 267 267 263 F F F <---
171982024-02-07T19:02:06.204Z 19 380-399 156 156 150 267 267 252 F F F <---
171992024-02-07T19:02:06.204Z 20 400-419 145 145 145 245 245 245 F F F
172002024-02-07T19:02:06.204Z 21 420-439 156 156 152 266 266 258 F F F <---
172012024-02-07T19:02:06.204Z 22 440-459 147 147 147 248 248 248 F F F
172022024-02-07T19:02:06.205Z 23 460-479 152 152 152 259 259 259 F F F
172032024-02-07T19:02:06.205Z 24 480-499 156 156 153 264 264 261 T T F <---
172042024-02-07T19:02:06.205Z 25 500-519 155 155 155 263 263 263 T T F <---
172052024-02-07T19:02:06.205Z 26 520-539 155 155 155 263 263 263 F F F
172062024-02-07T19:02:06.205Z 27 540-559 152 152 152 258 258 258 F F F
172072024-02-07T19:02:06.205Z 28 560-579 156 156 155 267 267 263 F F F <---
172082024-02-07T19:02:06.205Z 29 580-599 150 150 150 254 254 254 F F F
172092024-02-07T19:02:06.205ZMax gen: 156, Max flush: 267
172102024-02-07T19:02:06.205ZError: Difference in extent metadata found!
172112024-02-07T19:02:06.205ZA Difference in extent metadata is expected here
172122024-02-07T19:02:06.205ZOn loop 78
172132024-02-07T19:02:06.205Z
172142024-02-07T19:02:06.209ZVerifying data now
172152024-02-07T19:02:10.754ZNo change after verify
172162024-02-07T19:02:10.755ZLoop: 78 Downstairs dump after verify (and repair):
172172024-02-07T19:02:10.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172182024-02-07T19:02:10.770Z 0 000-019 151 151 151 255 255 255 F F F
172192024-02-07T19:02:10.770Z 1 020-039 156 156 156 264 264 264 F F F
172202024-02-07T19:02:10.770Z 2 040-059 147 147 147 248 248 248 F F F
172212024-02-07T19:02:10.770Z 3 060-079 155 155 155 263 263 263 F F F
172222024-02-07T19:02:10.770Z 4 080-099 150 150 150 253 253 253 F F F
172232024-02-07T19:02:10.770Z 5 100-119 152 152 152 260 260 260 F F F
172242024-02-07T19:02:10.771Z 6 120-139 155 155 155 263 263 263 F F F
172252024-02-07T19:02:10.771Z 7 140-159 156 156 156 267 267 267 F F F
172262024-02-07T19:02:10.771Z 8 160-179 154 154 154 262 262 262 F F F
172272024-02-07T19:02:10.771Z 9 180-199 153 153 153 261 261 261 F F F
172282024-02-07T19:02:10.771Z 10 200-219 157 157 157 268 268 268 F F F
172292024-02-07T19:02:10.771Z 11 220-239 155 155 155 263 263 263 F F F
172302024-02-07T19:02:10.771Z 12 240-259 155 155 155 263 263 263 F F F
172312024-02-07T19:02:10.771Z 13 260-279 155 155 155 263 263 263 F F F
172322024-02-07T19:02:10.771Z 14 280-299 155 155 155 263 263 263 F F F
172332024-02-07T19:02:10.771Z 15 300-319 149 149 149 251 251 251 F F F
172342024-02-07T19:02:10.777Z 16 320-339 156 156 156 267 267 267 F F F
172352024-02-07T19:02:10.777Z 17 340-359 156 156 156 267 267 267 F F F
172362024-02-07T19:02:10.777Z 18 360-379 156 156 156 267 267 267 F F F
172372024-02-07T19:02:10.777Z 19 380-399 156 156 156 267 267 267 F F F
172382024-02-07T19:02:10.777Z 20 400-419 145 145 145 245 245 245 F F F
172392024-02-07T19:02:10.777Z 21 420-439 156 156 156 266 266 266 F F F
172402024-02-07T19:02:10.777Z 22 440-459 147 147 147 248 248 248 F F F
172412024-02-07T19:02:10.777Z 23 460-479 152 152 152 259 259 259 F F F
172422024-02-07T19:02:10.778Z 24 480-499 157 157 157 268 268 268 F F F
172432024-02-07T19:02:10.778Z 25 500-519 157 157 157 268 268 268 F F F
172442024-02-07T19:02:10.778Z 26 520-539 155 155 155 263 263 263 F F F
172452024-02-07T19:02:10.778Z 27 540-559 152 152 152 258 258 258 F F F
172462024-02-07T19:02:10.778Z 28 560-579 156 156 156 267 267 267 F F F
172472024-02-07T19:02:10.778Z 29 580-599 150 150 150 254 254 254 F F F
172482024-02-07T19:02:10.778ZMax gen: 157, Max flush: 268
172492024-02-07T19:02:10.778Z
172502024-02-07T19:02:10.778ZBegin loop 79
172512024-02-07T19:02:10.778ZDownstairs to restart: 1
172522024-02-07T19:02:10.791Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.996859777Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767}
172532024-02-07T19:02:10.791Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997102744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767}
172542024-02-07T19:02:10.791Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997122151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767}
172552024-02-07T19:02:10.797Z{"msg":"Crucible cc1c6823-866c-4f46-a23b-cb23af579d7d has session id: e1b88ae7-2ac5-4736-81b8-766a3b244d23","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997155357Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767Crucible runtime is spawned
172562024-02-07T19:02:10.797Z,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172572024-02-07T19:02:10.797Z{"msg":"Upstairs opts: Upstairs UUID: cc1c6823-866c-4f46-a23b-cb23af579d7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997209209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172582024-02-07T19:02:10.797Z{"msg":"Crucible stats registered with UUID: cc1c6823-866c-4f46-a23b-cb23af579d7d","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997245535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172592024-02-07T19:02:10.797Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997282899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767}
172602024-02-07T19:02:10.797Z{"msg":"cc1c6823-866c-4f46-a23b-cb23af579d7d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997483332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172612024-02-07T19:02:10.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997511289Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172622024-02-07T19:02:10.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997526966Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172632024-02-07T19:02:10.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997556753Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172642024-02-07T19:02:10.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.99757869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172652024-02-07T19:02:10.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997600776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172662024-02-07T19:02:10.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.99764831Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172672024-02-07T19:02:10.797Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997677496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172682024-02-07T19:02:10.797Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997703133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172692024-02-07T19:02:10.797Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997738548Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172702024-02-07T19:02:10.797Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:09.997865911Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172712024-02-07T19:02:10.797Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:09.997924383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172722024-02-07T19:02:10.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997952689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172732024-02-07T19:02:10.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.997989064Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172742024-02-07T19:02:10.798Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:09.99802015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172752024-02-07T19:02:10.798Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.998064414Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172762024-02-07T19:02:10.798Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.99808965Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172772024-02-07T19:02:10.798Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.998135364Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172782024-02-07T19:02:10.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.998588483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172792024-02-07T19:02:10.798Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.998630017Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172802024-02-07T19:02:10.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.998645935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172812024-02-07T19:02:10.798Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.998702947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172822024-02-07T19:02:10.798Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.999302706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172832024-02-07T19:02:10.798Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.999340831Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172842024-02-07T19:02:10.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.999887917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172852024-02-07T19:02:10.798Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.999920533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172862024-02-07T19:02:10.798Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.999935981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172872024-02-07T19:02:10.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:09.999967216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172882024-02-07T19:02:10.798Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:10.000011621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172892024-02-07T19:02:10.798Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:10.000028738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172902024-02-07T19:02:11.793Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:10.998656933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172912024-02-07T19:02:11.799Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:10.998809572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172922024-02-07T19:02:11.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:10.999589716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172932024-02-07T19:02:11.799Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:10.999625242Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172942024-02-07T19:02:11.799Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000164109Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172952024-02-07T19:02:11.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000731982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172962024-02-07T19:02:11.799Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000774566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172972024-02-07T19:02:11.799Z{"msg":"[0]R flush_numbers[0..12]: [255, 264, 248, 263, 253, 260, 263, 267, 262, 261, 268, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000801983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172982024-02-07T19:02:11.799Z{"msg":"[0]R generation[0..12]: [151, 156, 147, 155, 150, 152, 155, 156, 154, 153, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000828949Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
172992024-02-07T19:02:11.799Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000855925Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173002024-02-07T19:02:11.799Z{"msg":"[1]R flush_numbers[0..12]: [255, 264, 248, 263, 253, 260, 263, 267, 262, 261, 268, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000882882Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173012024-02-07T19:02:11.799Z{"msg":"[1]R generation[0..12]: [151, 156, 147, 155, 150, 152, 155, 156, 154, 153, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000909068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173022024-02-07T19:02:11.799Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.000935545Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
173032024-02-07T19:02:11.799Zsession_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173042024-02-07T19:02:11.799Z{"msg":"[2]R flush_numbers[0..12]: [255, 264, 248, 263, 253, 260, 263, 267, 262, 261, 268, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.00096533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173052024-02-07T19:02:11.799Z{"msg":"[2]R generation[0..12]: [151, 156, 147, 155, 150, 152, 155, 156, 154, 153, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001001545Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173062024-02-07T19:02:11.799Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001028972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173072024-02-07T19:02:11.799Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001061708Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173082024-02-07T19:02:11.799Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001087164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173092024-02-07T19:02:11.800Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.00111406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173102024-02-07T19:02:11.800Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001137817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173112024-02-07T19:02:11.800Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001166193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173122024-02-07T19:02:11.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.00118979Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173132024-02-07T19:02:11.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001217366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173142024-02-07T19:02:11.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001241493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173152024-02-07T19:02:11.800Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.00126832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173162024-02-07T19:02:11.800Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001292026Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173172024-02-07T19:02:11.800Z{"msg":"cc1c6823-866c-4f46-a23b-cb23af579d7d is now active with session: e1b88ae7-2ac5-4736-81b8-766a3b244d23","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001319373Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173182024-02-07T19:02:11.800Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.00134136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767,"session_id":"e1b88ae7-2ac5-4736-81b8-766a3b244d23"}
173192024-02-07T19:02:11.800Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:11.001367716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767}
173202024-02-07T19:02:11.800ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173212024-02-07T19:02:11.800ZLoading write count information from file "/tmp/test_repair_verify.data"
173222024-02-07T19:02:11.800ZRun Repair workload
173232024-02-07T19:02:11.800Z01/30 Write block 41 len 8192 data: 9 8
173242024-02-07T19:02:11.803Z02/30 Read block 375 len 24576
173252024-02-07T19:02:11.803Z03/30 Read block 246 len 24576
173262024-02-07T19:02:11.804Z04/30 Write block 489 len 12288 data: 10 10 10
173272024-02-07T19:02:11.804Z05/30 Read block 312 len 20480
173282024-02-07T19:02:11.804Z06/30 Read block 549 len 32768
173292024-02-07T19:02:11.804Z07/30 Read block 579 len 12288
173302024-02-07T19:02:11.804Z08/30 Write block 389 len 20480 data: 14 8 7 7 10
173312024-02-07T19:02:11.804Z09/30 Read block 187 len 24576
173322024-02-07T19:02:11.807Z10/30 Flush
173332024-02-07T19:02:11.807Z11/30 Flush
173342024-02-07T19:02:11.811Z12/30 Write block 326 len 12288 data: 9 8 9
173352024-02-07T19:02:11.811Z13/30 Flush
173362024-02-07T19:02:11.811Z14/30 Write block 331 len 40960 data: 13 12 13 13 12 12 11 13 16 12
173372024-02-07T19:02:11.811Z15/30 Read block 413 len 28672
173382024-02-07T19:02:11.817Z16/30 Write block 131 len 20480 data: 11 10 10 10 8
173392024-02-07T19:02:11.817Z17/30 Read block 507 len 12288
173402024-02-07T19:02:11.817Z18/30 Read block 327 len 20480
173412024-02-07T19:02:11.821Z19/30 Write block 254 len 8192 data: 12 10
173422024-02-07T19:02:11.821Z20/30 Write block 477 len 8192 data: 11 10
173432024-02-07T19:02:11.821Z21/30 Write block 73 len 32768 data: 9 8 7 11 13 16 15 13
173442024-02-07T19:02:11.821Z22/30 Flush
173452024-02-07T19:02:11.821Z23/30 Write block 565 len 4096 data: 17
173462024-02-07T19:02:11.821Z24/30 Read block 561 len 36864
173472024-02-07T19:02:11.824Z25/30 Write block 76 len 28672 data: 12 14 17 16 14 10 10
173482024-02-07T19:02:11.824Z26/30 Write block 511 len 20480 data: 13 17 15 18 18
173492024-02-07T19:02:11.824Z27/30 Read block 224 len 16384
173502024-02-07T19:02:11.824Z28/30 Read block 453 len 32768
173512024-02-07T19:02:11.828Z29/30 Write block 364 len 36864 data: 11 10 14 14 15 15 17 16 12
173522024-02-07T19:02:11.828Z30/30 Write block 550 len 36864 data: 15 12 10 10 9 11 10 8 10
173532024-02-07T19:02:11.828Z----------------------------------------------------------------
173542024-02-07T19:02:11.828Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
173552024-02-07T19:02:11.828ZStates: Active Active Active
173562024-02-07T19:02:11.828ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
173572024-02-07T19:02:11.828Z 1 Acked 1000 Write 2 Done Done Done false
173582024-02-07T19:02:11.828Z 2 Acked 1001 Read 6 Done Sent Done false
173592024-02-07T19:02:11.828Z 3 Acked 1002 Read 6 Done Sent Done false
173602024-02-07T19:02:11.828Z 4 Acked 1003 Write 3 Done Sent Done false
173612024-02-07T19:02:11.829Z 5 Acked 1004 Read 5 Done Sent Done false
173622024-02-07T19:02:11.829Z 6 Acked 1005 Read 8 Done Sent Done false
173632024-02-07T19:02:11.829Z 7 Acked 1006 Read 3 Done Sent Done false
173642024-02-07T19:02:11.829Z 8 Acked 1007 Write 5 Done Sent Done false
173652024-02-07T19:02:11.829Z 9 Acked 1008 Read 6 Done Sent Done false
173662024-02-07T19:02:11.829Z 10 Acked 1009 Flush 0 Done Sent Done false
173672024-02-07T19:02:11.829Z 11 Acked 1010 Flush 0 Done Sent Done false
173682024-02-07T19:02:11.829Z 12 Acked 1011 Write 3 Done Sent Done false
173692024-02-07T19:02:11.829Z 13 Acked 1012 Flush 0 Done Sent Done false
173702024-02-07T19:02:11.829Z 14 Acked 1013 Write 10 Done Sent Done false
173712024-02-07T19:02:11.829Z 15 Acked 1014 Read 7 Done Sent Done false
173722024-02-07T19:02:11.829Z 16 Acked 1015 Write 5 Done Sent Done false
173732024-02-07T19:02:11.829Z 17 Acked 1016 Read 3 Done Sent Done false
173742024-02-07T19:02:11.829Z 18 Acked 1017 Read 5 Done Sent Done false
173752024-02-07T19:02:11.829Z 19 Acked 1018 Write 2 Done Sent Done false
173762024-02-07T19:02:11.829Z 20 Acked 1019 Write 2 Done Sent Done false
173772024-02-07T19:02:11.829Z 21 Acked 1020 Write 8 Done Sent Done false
173782024-02-07T19:02:11.829Z 22 Acked 1021 Flush 0 Done Sent Done false
173792024-02-07T19:02:11.829Z 23 Acked 1022 Write 1 Done Sent Done false
173802024-02-07T19:02:11.829Z 24 Acked 1023 Read 9 Done Sent Done false
173812024-02-07T19:02:11.829Z 25 Acked 1024 Write 7 Done Sent Done false
173822024-02-07T19:02:11.829Z 26 Acked 1025 Write 5 Done Sent Done false
173832024-02-07T19:02:11.829Z 27 Acked 1026 Read 4 Done Sent Done false
173842024-02-07T19:02:11.829Z 28 Acked 1027 Read 8 Done Sent Done false
173852024-02-07T19:02:11.829Z 29 Acked 1028 Write 9 Done Sent Done false
173862024-02-07T19:02:11.830Z 30 Acked 1029 Write 9 Sent Sent Sent false
173872024-02-07T19:02:11.830Z STATES DS:0 DS:1 DS:2 TOTAL
173882024-02-07T19:02:11.830Z New 0 0 0 0
173892024-02-07T19:02:11.830Z Sent 1 29 1 31
173902024-02-07T19:02:11.830Z Done 29 1 29 59
173912024-02-07T19:02:11.830Z Skipped 0 0 0 0
173922024-02-07T19:02:11.830Z Error 0 0 0 0
173932024-02-07T19:02:11.830ZLast Flush: 1021 0 1021
173942024-02-07T19:02:11.830ZDownstairs last five completed:
173952024-02-07T19:02:11.830ZUpstairs last five completed: 30 29 28 27 26
173962024-02-07T19:02:11.830Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:11.032400849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1767}
173972024-02-07T19:02:11.830ZWrote out file "/tmp/test_repair_verify.data"
173982024-02-07T19:02:11.833Z
173992024-02-07T19:02:11.836ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174002024-02-07T19:02:11.861ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174012024-02-07T19:02:11.861Z 0 000-019 151 151 151 255 255 255 F F F
174022024-02-07T19:02:11.861Z 1 020-039 156 156 156 264 264 264 F F F
174032024-02-07T19:02:11.861Z 2 040-059 158 147 158 269 248 269 F T F <---
174042024-02-07T19:02:11.861Z 3 060-079 158 155 158 272 263 272 T F T <---
174052024-02-07T19:02:11.865Z 4 080-099 158 150 158 272 253 272 T F T <---
174062024-02-07T19:02:11.865Z 5 100-119 152 152 152 260 260 260 F F F
174072024-02-07T19:02:11.865Z 6 120-139 158 155 158 272 263 272 F F F <---
174082024-02-07T19:02:11.865Z 7 140-159 156 156 156 267 267 267 F F F
174092024-02-07T19:02:11.865Z 8 160-179 154 154 154 262 262 262 F F F
174102024-02-07T19:02:11.865Z 9 180-199 153 153 153 261 261 261 F F F
174112024-02-07T19:02:11.865Z 10 200-219 157 157 157 268 268 268 F F F
174122024-02-07T19:02:11.865Z 11 220-239 155 155 155 263 263 263 F F F
174132024-02-07T19:02:11.865Z 12 240-259 158 155 158 272 263 272 F F F <---
174142024-02-07T19:02:11.865Z 13 260-279 155 155 155 263 263 263 F F F
174152024-02-07T19:02:11.865Z 14 280-299 155 155 155 263 263 263 F F F
174162024-02-07T19:02:11.865Z 15 300-319 149 149 149 251 251 251 F F F
174172024-02-07T19:02:11.865Z 16 320-339 158 156 158 272 267 272 F F F <---
174182024-02-07T19:02:11.865Z 17 340-359 158 156 158 272 267 272 F F F <---
174192024-02-07T19:02:11.865Z 18 360-379 156 156 156 267 267 267 T F T <---
174202024-02-07T19:02:11.866Z 19 380-399 158 156 158 269 267 269 F F F <---
174212024-02-07T19:02:11.866Z 20 400-419 145 145 145 245 245 245 F F F
174222024-02-07T19:02:11.866Z 21 420-439 156 156 156 266 266 266 F F F
174232024-02-07T19:02:11.866Z 22 440-459 147 147 147 248 248 248 F F F
174242024-02-07T19:02:11.866Z 23 460-479 158 152 158 272 259 272 F F F <---
174252024-02-07T19:02:11.866Z 24 480-499 158 157 158 269 268 269 F F F <---
174262024-02-07T19:02:11.866Z 25 500-519 157 157 157 268 268 268 T F T <---
174272024-02-07T19:02:11.866Z 26 520-539 155 155 155 263 263 263 F F F
174282024-02-07T19:02:11.866Z 27 540-559 152 152 152 258 258 258 T F T <---
174292024-02-07T19:02:11.866Z 28 560-579 156 156 156 267 267 267 T F T <---
174302024-02-07T19:02:11.866Z 29 580-599 150 150 150 254 254 254 F F F
174312024-02-07T19:02:11.866ZMax gen: 158, Max flush: 272
174322024-02-07T19:02:11.866ZError: Difference in extent metadata found!
174332024-02-07T19:02:11.866ZA Difference in extent metadata is expected here
174342024-02-07T19:02:11.866ZOn loop 79
174352024-02-07T19:02:11.866Z
174362024-02-07T19:02:11.866ZVerifying data now
174372024-02-07T19:02:16.568ZNo change after verify
174382024-02-07T19:02:16.568ZLoop: 79 Downstairs dump after verify (and repair):
174392024-02-07T19:02:16.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174402024-02-07T19:02:16.583Z 0 000-019 151 151 151 255 255 255 F F F
174412024-02-07T19:02:16.583Z 1 020-039 156 156 156 264 264 264 F F F
174422024-02-07T19:02:16.584Z 2 040-059 158 158 158 269 269 269 F F F
174432024-02-07T19:02:16.584Z 3 060-079 159 159 159 273 273 273 F F F
174442024-02-07T19:02:16.584Z 4 080-099 159 159 159 273 273 273 F F F
174452024-02-07T19:02:16.584Z 5 100-119 152 152 152 260 260 260 F F F
174462024-02-07T19:02:16.584Z 6 120-139 158 158 158 272 272 272 F F F
174472024-02-07T19:02:16.584Z 7 140-159 156 156 156 267 267 267 F F F
174482024-02-07T19:02:16.584Z 8 160-179 154 154 154 262 262 262 F F F
174492024-02-07T19:02:16.584Z 9 180-199 153 153 153 261 261 261 F F F
174502024-02-07T19:02:16.587Z 10 200-219 157 157 157 268 268 268 F F F
174512024-02-07T19:02:16.587Z 11 220-239 155 155 155 263 263 263 F F F
174522024-02-07T19:02:16.587Z 12 240-259 158 158 158 272 272 272 F F F
174532024-02-07T19:02:16.587Z 13 260-279 155 155 155 263 263 263 F F F
174542024-02-07T19:02:16.588Z 14 280-299 155 155 155 263 263 263 F F F
174552024-02-07T19:02:16.588Z 15 300-319 149 149 149 251 251 251 F F F
174562024-02-07T19:02:16.588Z 16 320-339 158 158 158 272 272 272 F F F
174572024-02-07T19:02:16.588Z 17 340-359 158 158 158 272 272 272 F F F
174582024-02-07T19:02:16.588Z 18 360-379 159 159 159 273 273 273 F F F
174592024-02-07T19:02:16.588Z 19 380-399 158 158 158 269 269 269 F F F
174602024-02-07T19:02:16.588Z 20 400-419 145 145 145 245 245 245 F F F
174612024-02-07T19:02:16.588Z 21 420-439 156 156 156 266 266 266 F F F
174622024-02-07T19:02:16.588Z 22 440-459 147 147 147 248 248 248 F F F
174632024-02-07T19:02:16.588Z 23 460-479 158 158 158 272 272 272 F F F
174642024-02-07T19:02:16.588Z 24 480-499 158 158 158 269 269 269 F F F
174652024-02-07T19:02:16.588Z 25 500-519 159 159 159 273 273 273 F F F
174662024-02-07T19:02:16.588Z 26 520-539 155 155 155 263 263 263 F F F
174672024-02-07T19:02:16.588Z 27 540-559 159 159 159 273 273 273 F F F
174682024-02-07T19:02:16.588Z 28 560-579 159 159 159 273 273 273 F F F
174692024-02-07T19:02:16.588Z 29 580-599 150 150 150 254 254 254 F F F
174702024-02-07T19:02:16.588ZMax gen: 159, Max flush: 273
174712024-02-07T19:02:16.588Z
174722024-02-07T19:02:16.588ZBegin loop 80
174732024-02-07T19:02:16.588ZDownstairs to restart: 0
174742024-02-07T19:02:16.604Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.80915555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775}
174752024-02-07T19:02:16.604Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809391748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775}
174762024-02-07T19:02:16.610Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809418264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775}
174772024-02-07T19:02:16.610Z{"msg":"Crucible efdb66c2-cdad-42ca-ab36-d174a396dcf0 has session id: 53380ac8-76b0-43ad-86de-9593784b0035","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809460369Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174782024-02-07T19:02:16.610Z{"Crucible runtime is spawned
174792024-02-07T19:02:16.610Zmsg":"Upstairs opts: Upstairs UUID: efdb66c2-cdad-42ca-ab36-d174a396dcf0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.80952794Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174802024-02-07T19:02:16.611Z{"msg":"Crucible stats registered with UUID: efdb66c2-cdad-42ca-ab36-d174a396dcf0","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809570684Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174812024-02-07T19:02:16.611Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.80959871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775}
174822024-02-07T19:02:16.611Z{"msg":"efdb66c2-cdad-42ca-ab36-d174a396dcf0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809730602Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174832024-02-07T19:02:16.611Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809794804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174842024-02-07T19:02:16.611Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809860284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174852024-02-07T19:02:16.611Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809910538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174862024-02-07T19:02:16.611Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.809976929Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174872024-02-07T19:02:16.611Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810048359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174882024-02-07T19:02:16.611Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810105111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174892024-02-07T19:02:16.611Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810153665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174902024-02-07T19:02:16.611Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810205458Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174912024-02-07T19:02:16.611Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810247512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174922024-02-07T19:02:16.611Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810269739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174932024-02-07T19:02:16.611Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810362907Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174942024-02-07T19:02:16.611Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810417319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174952024-02-07T19:02:16.611Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810827744Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174962024-02-07T19:02:16.611Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810866119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174972024-02-07T19:02:16.611Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.810956486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174982024-02-07T19:02:16.611Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811024197Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
174992024-02-07T19:02:16.611Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811366741Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175002024-02-07T19:02:16.611Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811509032Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175012024-02-07T19:02:16.611Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811673849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175022024-02-07T19:02:16.612Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811756298Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175032024-02-07T19:02:16.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811934444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175042024-02-07T19:02:16.612Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.811979098Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175052024-02-07T19:02:16.612Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812006524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175062024-02-07T19:02:16.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.81211303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175072024-02-07T19:02:16.612Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812142926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175082024-02-07T19:02:16.612Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812158354Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175092024-02-07T19:02:16.612Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812357527Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175102024-02-07T19:02:16.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812824664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175112024-02-07T19:02:16.612Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.81285419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175122024-02-07T19:02:16.612Z{"msg":"[0]R flush_numbers[0..12]: [255, 264, 269, 273, 273, 260, 272, 267, 262, 261, 268, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812870828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175132024-02-07T19:02:16.612Z{"msg":"[0]R generation[0..12]: [151, 156, 158, 159, 159, 152, 158, 156, 154, 153, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812891714Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175142024-02-07T19:02:16.612Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812906552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175152024-02-07T19:02:16.612Z{"msg":"[1]R flush_numbers[0..12]: [255, 264, 269, 273, 273, 260, 272, 267, 262, 261, 268, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.8129278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175162024-02-07T19:02:16.612Z{"msg":"[1]R generation[0..12]: [151, 156, 158, 159, 159, 152, 158, 156, 154, 153, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812942528Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175172024-02-07T19:02:16.612Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812964165Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175182024-02-07T19:02:16.618Z{"msg":"[2]R flush_numbers[0..12]: [255, 264, 269, 273, 273, 260, 272, 267, 262, 261, 268, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.812978903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175192024-02-07T19:02:16.618Z{"msg":"[2]R generation[0..12]: [151, 156, 158, 159, 159, 152, 158, 156, 154, 153, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813004859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175202024-02-07T19:02:16.618Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
175212024-02-07T19:02:16.618Z2024-02-07T19:02:15.813019627Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175222024-02-07T19:02:16.618Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813036215Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175232024-02-07T19:02:16.618Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813056592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175242024-02-07T19:02:16.618Z{"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813070101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175252024-02-07T19:02:16.618Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813092138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175262024-02-07T19:02:16.618Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813106556Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175272024-02-07T19:02:16.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813126493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175282024-02-07T19:02:16.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813140191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175292024-02-07T19:02:16.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813160148Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175302024-02-07T19:02:16.618Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813173907Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175312024-02-07T19:02:16.618Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813192634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175322024-02-07T19:02:16.619Z{"msg":"efdb66c2-cdad-42ca-ab36-d174a396dcf0 is now active with session: 53380ac8-76b0-43ad-86de-9593784b0035","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813205922Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175332024-02-07T19:02:16.619Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.81322433Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775,"session_id":"53380ac8-76b0-43ad-86de-9593784b0035"}
175342024-02-07T19:02:16.619Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:15.813237188Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775}
175352024-02-07T19:02:16.619ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
175362024-02-07T19:02:16.619ZLoading write count information from file "/tmp/test_repair_verify.data"
175372024-02-07T19:02:16.619ZRun Repair workload
175382024-02-07T19:02:16.619Z01/30 Write block 95 len 28672 data: 16 17 15 13 12 10 9
175392024-02-07T19:02:16.619Z02/30 Read block 368 len 36864
175402024-02-07T19:02:16.619Z03/30 Read block 100 len 24576
175412024-02-07T19:02:16.619Z04/30 Write block 548 len 36864 data: 16 17 16 13 11 11 10 12 11
175422024-02-07T19:02:16.619Z05/30 Write block 281 len 40960 data: 13 12 9 15 15 16 19 18 17 19
175432024-02-07T19:02:16.619Z06/30 Write block 343 len 12288 data: 11 12 15
175442024-02-07T19:02:16.623Z07/30 Read block 391 len 16384
175452024-02-07T19:02:16.623Z08/30 Read block 166 len 16384
175462024-02-07T19:02:16.623Z09/30 Write block 32 len 36864 data: 18 17 18 16 16 11 10 8 9
175472024-02-07T19:02:16.623Z10/30 Read block 405 len 8192
175482024-02-07T19:02:16.623Z11/30 Read block 40 len 36864
175492024-02-07T19:02:16.623Z12/30 Read block 376 len 32768
175502024-02-07T19:02:16.626Z13/30 Write block 437 len 28672 data: 12 8 11 12 12 13 14
175512024-02-07T19:02:16.626Z14/30 Read block 46 len 40960
175522024-02-07T19:02:16.626Z15/30 Write block 205 len 16384 data: 13 10 10 13
175532024-02-07T19:02:16.626Z16/30 Write block 572 len 36864 data: 11 11 8 9 9 9 9 10 11
175542024-02-07T19:02:16.629Z17/30 Write block 464 len 28672 data: 9 9 9 6 5 7 9
175552024-02-07T19:02:16.629Z18/30 Write block 272 len 36864 data: 9 10 8 10 9 9 11 11 12
175562024-02-07T19:02:16.629Z19/30 Flush
175572024-02-07T19:02:16.632Z20/30 Write block 527 len 4096 data: 11
175582024-02-07T19:02:16.636Z21/30 Flush
175592024-02-07T19:02:16.636Z22/30 Write block 478 len 4096 data: 11
175602024-02-07T19:02:16.636Z23/30 Write block 492 len 32768 data: 12 11 15 9 13 10 12 11
175612024-02-07T19:02:16.639Z24/30 Read block 67 len 20480
175622024-02-07T19:02:16.639Z25/30 Read block 95 len 4096
175632024-02-07T19:02:16.639Z26/30 Write block 264 len 36864 data: 17 14 14 15 12 11 8 9 10
175642024-02-07T19:02:16.645Z27/30 Write block 305 len 24576 data: 9 8 7 10 11 11
175652024-02-07T19:02:16.645Z28/30 Read block 379 len 12288
175662024-02-07T19:02:16.645Z29/30 Write block 150 len 36864 data: 15 14 12 15 15 12 12 10 13
175672024-02-07T19:02:16.645Z30/30 Write block 339 len 36864 data: 17 13 11 11 12 13 16 17 17
175682024-02-07T19:02:16.645Z----------------------------------------------------------------
175692024-02-07T19:02:16.645Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
175702024-02-07T19:02:16.645ZStates: Active Active Active
175712024-02-07T19:02:16.645ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
175722024-02-07T19:02:16.645Z 1 Acked 1000 Write 7 Done Done Done false
175732024-02-07T19:02:16.645Z 2 Acked 1001 Read 9 Done Done Done false
175742024-02-07T19:02:16.646Z 3 Acked 1002 Read 6 Sent Done Done false
175752024-02-07T19:02:16.646Z 4 Acked 1003 Write 9 Sent Done Done false
175762024-02-07T19:02:16.646Z 5 Acked 1004 Write 10 Sent Done Done false
175772024-02-07T19:02:16.646Z 6 Acked 1005 Write 3 Sent Done Done false
175782024-02-07T19:02:16.646Z 7 Acked 1006 Read 4 Sent Done Done false
175792024-02-07T19:02:16.646Z 8 Acked 1007 Read 4 Sent Done Done false
175802024-02-07T19:02:16.646Z 9 Acked 1008 Write 9 Sent Done Done false
175812024-02-07T19:02:16.646Z 10 Acked 1009 Read 2 Sent Done Done false
175822024-02-07T19:02:16.646Z 11 Acked 1010 Read 9 Sent Done Done false
175832024-02-07T19:02:16.646Z 12 Acked 1011 Read 8 Sent Done Done false
175842024-02-07T19:02:16.646Z 13 Acked 1012 Write 7 Sent Done Done false
175852024-02-07T19:02:16.646Z 14 Acked 1013 Read 10 Sent Done Done false
175862024-02-07T19:02:16.646Z 15 Acked 1014 Write 4 Sent Done Done false
175872024-02-07T19:02:16.646Z 16 Acked 1015 Write 9 Sent Done Done false
175882024-02-07T19:02:16.646Z 17 Acked 1016 Write 7 Sent Done Done false
175892024-02-07T19:02:16.646Z 18 Acked 1017 Write 9 Sent Done Done false
175902024-02-07T19:02:16.646Z 19 Acked 1018 Flush 0 Sent Done Done false
175912024-02-07T19:02:16.646Z 20 Acked 1019 Write 1 Sent Done Done false
175922024-02-07T19:02:16.646Z 21 Acked 1020 Flush 0 Sent Done Done false
175932024-02-07T19:02:16.646Z 22 Acked 1021 Write 1 Sent Done Done false
175942024-02-07T19:02:16.646Z 23 Acked 1022 Write 8 Sent Done Done false
175952024-02-07T19:02:16.646Z 24 Acked 1023 Read 5 Sent Done Done false
175962024-02-07T19:02:16.646Z 25 Acked 1024 Read 1 Sent Done Done false
175972024-02-07T19:02:16.646Z 26 Acked 1025 Write 9 Sent Done Done false
175982024-02-07T19:02:16.646Z 27 Acked 1026 Write 6 Sent Done Done false
175992024-02-07T19:02:16.647Z 28 Acked 1027 Read 3 Sent Done Done false
176002024-02-07T19:02:16.647Z 29 Acked 1028 Write 9 Sent Done Sent false
176012024-02-07T19:02:16.647Z 30 Acked 1029 Write 9 Sent Sent Sent false
176022024-02-07T19:02:16.650Z STATES DS:0 DS:1 DS:2 TOTAL
176032024-02-07T19:02:16.650Z New 0 0 0 0
176042024-02-07T19:02:16.650Z Sent 28 1 2 31
176052024-02-07T19:02:16.650Z Done 2 29 28 59
176062024-02-07T19:02:16.650Z Skipped 0 0 0 0
176072024-02-07T19:02:16.651Z Error 0 0 0 0
176082024-02-07T19:02:16.651ZLast Flush: 0 1020 1020
176092024-02-07T19:02:16.651ZDownstairs last five completed:
176102024-02-07T19:02:16.651ZUpstairs last five completed: 30 29 28 27 26
176112024-02-07T19:02:16.651Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:15.847514932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1775}
176122024-02-07T19:02:16.651ZWrote out file "/tmp/test_repair_verify.data"
176132024-02-07T19:02:16.651Z
176142024-02-07T19:02:16.655ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176152024-02-07T19:02:16.670ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176162024-02-07T19:02:16.670Z 0 000-019 151 151 151 255 255 255 F F F
176172024-02-07T19:02:16.671Z 1 020-039 156 160 160 264 274 274 F F F <---
176182024-02-07T19:02:16.671Z 2 040-059 158 160 160 269 274 274 F F F <---
176192024-02-07T19:02:16.671Z 3 060-079 159 159 159 273 273 273 F F F
176202024-02-07T19:02:16.671Z 4 080-099 159 160 160 273 274 274 T F F <---
176212024-02-07T19:02:16.671Z 5 100-119 152 160 160 260 274 274 T F F <---
176222024-02-07T19:02:16.671Z 6 120-139 158 158 158 272 272 272 F F F
176232024-02-07T19:02:16.671Z 7 140-159 156 156 156 267 267 267 F T T <---
176242024-02-07T19:02:16.671Z 8 160-179 154 154 154 262 262 262 F F F
176252024-02-07T19:02:16.676Z 9 180-199 153 153 153 261 261 261 F F F
176262024-02-07T19:02:16.676Z 10 200-219 157 160 160 268 274 274 F F F <---
176272024-02-07T19:02:16.676Z 11 220-239 155 155 155 263 263 263 F F F
176282024-02-07T19:02:16.676Z 12 240-259 158 158 158 272 272 272 F F F
176292024-02-07T19:02:16.677Z 13 260-279 155 160 160 263 274 274 F T T <---
176302024-02-07T19:02:16.677Z 14 280-299 155 160 160 263 274 274 F F F <---
176312024-02-07T19:02:16.677Z 15 300-319 149 149 149 251 251 251 F T T <---
176322024-02-07T19:02:16.677Z 16 320-339 158 158 158 272 272 272 F T T <---
176332024-02-07T19:02:16.677Z 17 340-359 158 160 160 272 274 274 F T T <---
176342024-02-07T19:02:16.677Z 18 360-379 159 159 159 273 273 273 F F F
176352024-02-07T19:02:16.677Z 19 380-399 158 158 158 269 269 269 F F F
176362024-02-07T19:02:16.677Z 20 400-419 145 145 145 245 245 245 F F F
176372024-02-07T19:02:16.677Z 21 420-439 156 160 160 266 274 274 F F F <---
176382024-02-07T19:02:16.677Z 22 440-459 147 160 160 248 274 274 F F F <---
176392024-02-07T19:02:16.677Z 23 460-479 158 160 160 272 274 274 F T T <---
176402024-02-07T19:02:16.677Z 24 480-499 158 158 158 269 269 269 F T T <---
176412024-02-07T19:02:16.677Z 25 500-519 159 159 159 273 273 273 F F F
176422024-02-07T19:02:16.678Z 26 520-539 155 160 160 263 275 275 F F F <---
176432024-02-07T19:02:16.678Z 27 540-559 159 160 160 273 274 274 F F F <---
176442024-02-07T19:02:16.678Z 28 560-579 159 160 160 273 274 274 F F F <---
176452024-02-07T19:02:16.678Z 29 580-599 150 160 160 254 274 274 F F F <---
176462024-02-07T19:02:16.678ZMax gen: 160, Max flush: 275
176472024-02-07T19:02:16.678ZError: Difference in extent metadata found!
176482024-02-07T19:02:16.678ZA Difference in extent metadata is expected here
176492024-02-07T19:02:16.678ZOn loop 80
176502024-02-07T19:02:16.678Z
176512024-02-07T19:02:16.678ZVerifying data now
176522024-02-07T19:02:22.550ZNo change after verify
176532024-02-07T19:02:22.550ZLoop: 80 Downstairs dump after verify (and repair):
176542024-02-07T19:02:22.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176552024-02-07T19:02:22.565Z 0 000-019 151 151 151 255 255 255 F F F
176562024-02-07T19:02:22.566Z 1 020-039 160 160 160 274 274 274 F F F
176572024-02-07T19:02:22.566Z 2 040-059 160 160 160 274 274 274 F F F
176582024-02-07T19:02:22.566Z 3 060-079 159 159 159 273 273 273 F F F
176592024-02-07T19:02:22.566Z 4 080-099 160 160 160 274 274 274 F F F
176602024-02-07T19:02:22.566Z 5 100-119 160 160 160 274 274 274 F F F
176612024-02-07T19:02:22.566Z 6 120-139 158 158 158 272 272 272 F F F
176622024-02-07T19:02:22.570Z 7 140-159 161 161 161 276 276 276 F F F
176632024-02-07T19:02:22.570Z 8 160-179 154 154 154 262 262 262 F F F
176642024-02-07T19:02:22.570Z 9 180-199 153 153 153 261 261 261 F F F
176652024-02-07T19:02:22.570Z 10 200-219 160 160 160 274 274 274 F F F
176662024-02-07T19:02:22.570Z 11 220-239 155 155 155 263 263 263 F F F
176672024-02-07T19:02:22.570Z 12 240-259 158 158 158 272 272 272 F F F
176682024-02-07T19:02:22.570Z 13 260-279 161 161 161 276 276 276 F F F
176692024-02-07T19:02:22.570Z 14 280-299 160 160 160 274 274 274 F F F
176702024-02-07T19:02:22.570Z 15 300-319 161 161 161 276 276 276 F F F
176712024-02-07T19:02:22.570Z 16 320-339 161 161 161 276 276 276 F F F
176722024-02-07T19:02:22.571Z 17 340-359 161 161 161 276 276 276 F F F
176732024-02-07T19:02:22.571Z 18 360-379 159 159 159 273 273 273 F F F
176742024-02-07T19:02:22.571Z 19 380-399 158 158 158 269 269 269 F F F
176752024-02-07T19:02:22.571Z 20 400-419 145 145 145 245 245 245 F F F
176762024-02-07T19:02:22.571Z 21 420-439 160 160 160 274 274 274 F F F
176772024-02-07T19:02:22.571Z 22 440-459 160 160 160 274 274 274 F F F
176782024-02-07T19:02:22.571Z 23 460-479 161 161 161 276 276 276 F F F
176792024-02-07T19:02:22.571Z 24 480-499 161 161 161 276 276 276 F F F
176802024-02-07T19:02:22.571Z 25 500-519 159 159 159 273 273 273 F F F
176812024-02-07T19:02:22.571Z 26 520-539 160 160 160 275 275 275 F F F
176822024-02-07T19:02:22.571Z 27 540-559 160 160 160 274 274 274 F F F
176832024-02-07T19:02:22.571Z 28 560-579 160 160 160 274 274 274 F F F
176842024-02-07T19:02:22.571Z 29 580-599 160 160 160 274 274 274 F F F
176852024-02-07T19:02:22.572ZMax gen: 161, Max flush: 276
176862024-02-07T19:02:22.572Z
176872024-02-07T19:02:22.572ZBegin loop 81
176882024-02-07T19:02:22.572ZDownstairs to restart: 2
176892024-02-07T19:02:22.588Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791483056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783}
176902024-02-07T19:02:22.588Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791756829Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783}
176912024-02-07T19:02:22.588Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791793284Z","hostname":"Crucible runtime is spawned
176922024-02-07T19:02:22.588Zip-10-150-1-187.us-west-2.compute.internal","pid":1783}
176932024-02-07T19:02:22.588Z{"msg":"Crucible f195e0ad-4628-4e9f-94f9-fd9e2d10cd4f has session id: bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791830929Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
176942024-02-07T19:02:22.593Z{"msg":"Upstairs opts: Upstairs UUID: f195e0ad-4628-4e9f-94f9-fd9e2d10cd4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791873653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
176952024-02-07T19:02:22.594Z{"msg":"Crucible stats registered with UUID: f195e0ad-4628-4e9f-94f9-fd9e2d10cd4f","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791908898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
176962024-02-07T19:02:22.594Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.791924766Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783}
176972024-02-07T19:02:22.594Z{"msg":"f195e0ad-4628-4e9f-94f9-fd9e2d10cd4f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792017134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
176982024-02-07T19:02:22.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.79204308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
176992024-02-07T19:02:22.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792081275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177002024-02-07T19:02:22.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.79212139Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177012024-02-07T19:02:22.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792144027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177022024-02-07T19:02:22.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792167053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177032024-02-07T19:02:22.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792222096Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177042024-02-07T19:02:22.594Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792238884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177052024-02-07T19:02:22.594Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792261561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177062024-02-07T19:02:22.594Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792284677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177072024-02-07T19:02:22.595Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792445926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177082024-02-07T19:02:22.595Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792493869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177092024-02-07T19:02:22.595Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:21.792695712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177102024-02-07T19:02:22.595Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:21.792750254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177112024-02-07T19:02:22.595Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:21.792823145Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177122024-02-07T19:02:22.595Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792867019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177132024-02-07T19:02:22.595Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792909113Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177142024-02-07T19:02:22.595Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.792939089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177152024-02-07T19:02:22.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.793158799Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177162024-02-07T19:02:22.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.793194045Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177172024-02-07T19:02:22.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.793219521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177182024-02-07T19:02:22.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.793239958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177192024-02-07T19:02:22.595Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.793761198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177202024-02-07T19:02:22.596Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.793915727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177212024-02-07T19:02:22.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.794360987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177222024-02-07T19:02:22.596Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.794398352Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177232024-02-07T19:02:22.596Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.794415029Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177242024-02-07T19:02:22.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.794435917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177252024-02-07T19:02:22.596Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.794450775Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177262024-02-07T19:02:22.596Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:21.794466253Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177272024-02-07T19:02:23.590Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.794288686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177282024-02-07T19:02:23.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.794380923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177292024-02-07T19:02:23.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.79521874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177302024-02-07T19:02:23.596Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.795262674Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177312024-02-07T19:02:23.596Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.795792462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177322024-02-07T19:02:23.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796406639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177332024-02-07T19:02:23.596Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796464971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177342024-02-07T19:02:23.596Z{"msg":"[0]R flush_numbers[0..12]: [255, 274, 274, 273, 274, 274, 272, 276, 262, 261, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796508015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177352024-02-07T19:02:23.596Z{"msg":"[0]R generation[0..12]: [151, 160, 160, 159, 160, 160, 158, 161, 154, 153, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796535562Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177362024-02-07T19:02:23.596Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796554099Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177372024-02-07T19:02:23.596Z{"msg":"[1]R flush_numbers[0..12]: [255, 274, 274, 273, 274, 274, 272, 276, 262, 261, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796575686Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177382024-02-07T19:02:23.596Z{"msg":"[1]R generation[0..12]: [151, 160, 160, 159, 160, 160, 158, 161, 154, 153, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796590124Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177392024-02-07T19:02:23.596Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.79662595Z","Wait for a query_work_queue command to finish before sending IO
177402024-02-07T19:02:23.596Zhostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177412024-02-07T19:02:23.596Z{"msg":"[2]R flush_numbers[0..12]: [255, 274, 274, 273, 274, 274, 272, 276, 262, 261, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796667704Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177422024-02-07T19:02:23.596Z{"msg":"[2]R generation[0..12]: [151, 160, 160, 159, 160, 160, 158, 161, 154, 153, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796700569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177432024-02-07T19:02:23.596Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796716547Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177442024-02-07T19:02:23.596Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796729945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177452024-02-07T19:02:23.596Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796752073Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177462024-02-07T19:02:23.596Z{"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796775939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177472024-02-07T19:02:23.596Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796812964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177482024-02-07T19:02:23.596Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796835061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177492024-02-07T19:02:23.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796861758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177502024-02-07T19:02:23.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796885674Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177512024-02-07T19:02:23.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796929169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177522024-02-07T19:02:23.597Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.796955985Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177532024-02-07T19:02:23.597Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.797022326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177542024-02-07T19:02:23.597Z{"msg":"f195e0ad-4628-4e9f-94f9-fd9e2d10cd4f is now active with session: bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.79706414Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177552024-02-07T19:02:23.597Z{"msg":"Set Active after no reconciliation","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
177562024-02-07T19:02:23.597Zname":"crucible","level":30,"time":"2024-02-07T19:02:22.797104995Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783,"session_id":"bc3aee4f-b1c9-4ef5-9e51-19fb91c2d490"}
177572024-02-07T19:02:23.597Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:22.79714115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783}
177582024-02-07T19:02:23.597ZLoading write count information from file "/tmp/test_repair_verify.data"
177592024-02-07T19:02:23.597ZRun Repair workload
177602024-02-07T19:02:23.600Z01/30 Write block 34 len 12288 data: 19 17 17
177612024-02-07T19:02:23.601Z02/30 Read block 209 len 8192
177622024-02-07T19:02:23.601Z03/30 Read block 98 len 24576
177632024-02-07T19:02:23.601Z04/30 Write block 559 len 8192 data: 12 13
177642024-02-07T19:02:23.601Z05/30 Read block 571 len 28672
177652024-02-07T19:02:23.601Z06/30 Read block 23 len 16384
177662024-02-07T19:02:23.601Z07/30 Write block 465 len 4096 data: 10
177672024-02-07T19:02:23.601Z08/30 Read block 227 len 32768
177682024-02-07T19:02:23.604Z09/30 Write block 121 len 36864 data: 12 15 16 17 17 16 14 16 17
177692024-02-07T19:02:23.604Z10/30 Flush
177702024-02-07T19:02:23.610Z11/30 Write block 315 len 16384 data: 14 16 13 13
177712024-02-07T19:02:23.610Z12/30 Write block 424 len 12288 data: 10 15 12
177722024-02-07T19:02:23.610Z13/30 Read block 421 len 28672
177732024-02-07T19:02:23.611Z14/30 Write block 361 len 32768 data: 13 15 14 12 11 15 15 16
177742024-02-07T19:02:23.613Z15/30 Write block 185 len 20480 data: 8 8 10 12 10
177752024-02-07T19:02:23.614Z16/30 Write block 54 len 16384 data: 11 13 13 11
177762024-02-07T19:02:23.614Z17/30 Read block 377 len 12288
177772024-02-07T19:02:23.614Z18/30 Write block 425 len 12288 data: 16 13 11
177782024-02-07T19:02:23.614Z19/30 Read block 308 len 40960
177792024-02-07T19:02:23.614Z20/30 Read block 450 len 24576
177802024-02-07T19:02:23.614Z21/30 Read block 592 len 24576
177812024-02-07T19:02:23.614Z22/30 Write block 14 len 16384 data: 14 12 14 17
177822024-02-07T19:02:23.617Z23/30 Write block 324 len 24576 data: 7 9 10 9 10 8
177832024-02-07T19:02:23.617Z24/30 Write block 331 len 40960 data: 14 13 14 14 13 13 12 14 18 14
177842024-02-07T19:02:23.617Z25/30 Read block 253 len 24576
177852024-02-07T19:02:23.617Z26/30 Read block 382 len 16384
177862024-02-07T19:02:23.621Z27/30 Read block 229 len 8192
177872024-02-07T19:02:23.621Z28/30 Read block 21 len 24576
177882024-02-07T19:02:23.621Z29/30 Write block 186 len 36864 data: 9 11 13 11 10 14 15 13 15
177892024-02-07T19:02:23.621Z30/30 Write block 256 len 4096 data: 9
177902024-02-07T19:02:23.621Z----------------------------------------------------------------
177912024-02-07T19:02:23.621Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
177922024-02-07T19:02:23.621ZStates: Active Active Active
177932024-02-07T19:02:23.621ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177942024-02-07T19:02:23.621Z 1 Acked 1000 Write 3 Done Done Done false
177952024-02-07T19:02:23.621Z 2 Acked 1001 Read 2 Done Done Done false
177962024-02-07T19:02:23.622Z 3 Acked 1002 Read 6 Done Done Done false
177972024-02-07T19:02:23.622Z 4 Acked 1003 Write 2 Done Done Done false
177982024-02-07T19:02:23.622Z 5 Acked 1004 Read 7 Done Done Done false
177992024-02-07T19:02:23.622Z 6 Acked 1005 Read 4 Done Done Done false
178002024-02-07T19:02:23.622Z 7 Acked 1006 Write 1 Done Done Done false
178012024-02-07T19:02:23.622Z 8 Acked 1007 Read 8 Done Done Done false
178022024-02-07T19:02:23.622Z 9 Acked 1008 Write 9 Done Done Sent false
178032024-02-07T19:02:23.622Z 10 Acked 1009 Flush 0 Done Done Sent false
178042024-02-07T19:02:23.622Z 11 Acked 1010 Write 4 Done Done Sent false
178052024-02-07T19:02:23.622Z 12 Acked 1011 Write 3 Done Done Sent false
178062024-02-07T19:02:23.622Z 13 Acked 1012 Read 7 Done Done Sent false
178072024-02-07T19:02:23.622Z 14 Acked 1013 Write 8 Done Done Sent false
178082024-02-07T19:02:23.622Z 15 Acked 1014 Write 5 Done Done Sent false
178092024-02-07T19:02:23.622Z 16 Acked 1015 Write 4 Done Done Sent false
178102024-02-07T19:02:23.622Z 17 Acked 1016 Read 3 Done Done Sent false
178112024-02-07T19:02:23.622Z 18 Acked 1017 Write 3 Done Done Sent false
178122024-02-07T19:02:23.622Z 19 Acked 1018 Read 10 Done Done Sent false
178132024-02-07T19:02:23.622Z 20 Acked 1019 Read 6 Done Done Sent false
178142024-02-07T19:02:23.622Z 21 Acked 1020 Read 6 Done Done Sent false
178152024-02-07T19:02:23.623Z 22 Acked 1021 Write 4 Done Done Sent false
178162024-02-07T19:02:23.623Z 23 Acked 1022 Write 6 Done Done Sent false
178172024-02-07T19:02:23.623Z 24 Acked 1023 Write 10 Done Done Sent false
178182024-02-07T19:02:23.623Z 25 Acked 1024 Read 6 Done Done Sent false
178192024-02-07T19:02:23.623Z 26 Acked 1025 Read 4 Done Done Sent false
178202024-02-07T19:02:23.623Z 27 Acked 1026 Read 2 Done Done Sent false
178212024-02-07T19:02:23.623Z 28 Acked 1027 Read 6 Done Done Sent false
178222024-02-07T19:02:23.623Z 29 Acked 1028 Write 9 Sent Sent Sent false
178232024-02-07T19:02:23.623Z 30 Acked 1029 Write 1 Sent Sent Sent false
178242024-02-07T19:02:23.623Z STATES DS:0 DS:1 DS:2 TOTAL
178252024-02-07T19:02:23.623Z New 0 0 0 0
178262024-02-07T19:02:23.623Z Sent 2 2 22 26
178272024-02-07T19:02:23.623Z Done 28 28 8 64
178282024-02-07T19:02:23.623Z Skipped 0 0 0 0
178292024-02-07T19:02:23.623Z Error 0 0 0 0
178302024-02-07T19:02:23.623ZLast Flush: 1009 1009 0
178312024-02-07T19:02:23.623ZDownstairs last five completed:
178322024-02-07T19:02:23.623ZUpstairs last five completed: 30 29 28 27 26
178332024-02-07T19:02:23.627Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:22.824657928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1783}
178342024-02-07T19:02:23.627ZWrote out file "/tmp/test_repair_verify.data"
178352024-02-07T19:02:23.627Z
178362024-02-07T19:02:23.630ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178372024-02-07T19:02:23.649ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178382024-02-07T19:02:23.650Z 0 000-019 151 151 151 255 255 255 T T F <---
178392024-02-07T19:02:23.650Z 1 020-039 162 162 160 277 277 274 F F T <---
178402024-02-07T19:02:23.650Z 2 040-059 160 160 160 274 274 274 T T F <---
178412024-02-07T19:02:23.650Z 3 060-079 159 159 159 273 273 273 F F F
178422024-02-07T19:02:23.650Z 4 080-099 160 160 160 274 274 274 F F F
178432024-02-07T19:02:23.650Z 5 100-119 160 160 160 274 274 274 F F F
178442024-02-07T19:02:23.650Z 6 120-139 162 162 158 277 277 272 F F F <---
178452024-02-07T19:02:23.650Z 7 140-159 161 161 161 276 276 276 F F F
178462024-02-07T19:02:23.650Z 8 160-179 154 154 154 262 262 262 F F F
178472024-02-07T19:02:23.650Z 9 180-199 153 153 153 261 261 261 T T F <---
178482024-02-07T19:02:23.650Z 10 200-219 160 160 160 274 274 274 F F F
178492024-02-07T19:02:23.650Z 11 220-239 155 155 155 263 263 263 F F F
178502024-02-07T19:02:23.651Z 12 240-259 158 158 158 272 272 272 T T F <---
178512024-02-07T19:02:23.651Z 13 260-279 161 161 161 276 276 276 F F F
178522024-02-07T19:02:23.651Z 14 280-299 160 160 160 274 274 274 F F F
178532024-02-07T19:02:23.660Z 15 300-319 161 161 161 276 276 276 T T F <---
178542024-02-07T19:02:23.660Z 16 320-339 161 161 161 276 276 276 T T F <---
178552024-02-07T19:02:23.660Z 17 340-359 161 161 161 276 276 276 T T F <---
178562024-02-07T19:02:23.660Z 18 360-379 159 159 159 273 273 273 T T F <---
178572024-02-07T19:02:23.660Z 19 380-399 158 158 158 269 269 269 F F F
178582024-02-07T19:02:23.660Z 20 400-419 145 145 145 245 245 245 F F F
178592024-02-07T19:02:23.661Z 21 420-439 160 160 160 274 274 274 T T F <---
178602024-02-07T19:02:23.661Z 22 440-459 160 160 160 274 274 274 F F F
178612024-02-07T19:02:23.661Z 23 460-479 162 162 161 277 277 276 F F T <---
178622024-02-07T19:02:23.661Z 24 480-499 161 161 161 276 276 276 F F F
178632024-02-07T19:02:23.661Z 25 500-519 159 159 159 273 273 273 F F F
178642024-02-07T19:02:23.661Z 26 520-539 160 160 160 275 275 275 F F F
178652024-02-07T19:02:23.661Z 27 540-559 162 162 160 277 277 274 F F T <---
178662024-02-07T19:02:23.661Z 28 560-579 162 162 160 277 277 274 F F T <---
178672024-02-07T19:02:23.661Z 29 580-599 160 160 160 274 274 274 F F F
178682024-02-07T19:02:23.661ZMax gen: 162, Max flush: 277
178692024-02-07T19:02:23.661ZError: Difference in extent metadata found!
178702024-02-07T19:02:23.661ZA Difference in extent metadata is expected here
178712024-02-07T19:02:23.661ZOn loop 81
178722024-02-07T19:02:23.661Z
178732024-02-07T19:02:23.661ZVerifying data now
178742024-02-07T19:02:29.352ZNo change after verify
178752024-02-07T19:02:29.352ZLoop: 81 Downstairs dump after verify (and repair):
178762024-02-07T19:02:29.368ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178772024-02-07T19:02:29.368Z 0 000-019 163 163 163 278 278 278 F F F
178782024-02-07T19:02:29.368Z 1 020-039 162 162 162 277 277 277 F F F
178792024-02-07T19:02:29.369Z 2 040-059 163 163 163 278 278 278 F F F
178802024-02-07T19:02:29.369Z 3 060-079 159 159 159 273 273 273 F F F
178812024-02-07T19:02:29.369Z 4 080-099 160 160 160 274 274 274 F F F
178822024-02-07T19:02:29.369Z 5 100-119 160 160 160 274 274 274 F F F
178832024-02-07T19:02:29.369Z 6 120-139 162 162 162 277 277 277 F F F
178842024-02-07T19:02:29.369Z 7 140-159 161 161 161 276 276 276 F F F
178852024-02-07T19:02:29.369Z 8 160-179 154 154 154 262 262 262 F F F
178862024-02-07T19:02:29.369Z 9 180-199 163 163 163 278 278 278 F F F
178872024-02-07T19:02:29.369Z 10 200-219 160 160 160 274 274 274 F F F
178882024-02-07T19:02:29.369Z 11 220-239 155 155 155 263 263 263 F F F
178892024-02-07T19:02:29.369Z 12 240-259 163 163 163 278 278 278 F F F
178902024-02-07T19:02:29.372Z 13 260-279 161 161 161 276 276 276 F F F
178912024-02-07T19:02:29.372Z 14 280-299 160 160 160 274 274 274 F F F
178922024-02-07T19:02:29.372Z 15 300-319 163 163 163 278 278 278 F F F
178932024-02-07T19:02:29.373Z 16 320-339 163 163 163 278 278 278 F F F
178942024-02-07T19:02:29.373Z 17 340-359 163 163 163 278 278 278 F F F
178952024-02-07T19:02:29.373Z 18 360-379 163 163 163 278 278 278 F F F
178962024-02-07T19:02:29.373Z 19 380-399 158 158 158 269 269 269 F F F
178972024-02-07T19:02:29.373Z 20 400-419 145 145 145 245 245 245 F F F
178982024-02-07T19:02:29.373Z 21 420-439 163 163 163 278 278 278 F F F
178992024-02-07T19:02:29.373Z 22 440-459 160 160 160 274 274 274 F F F
179002024-02-07T19:02:29.373Z 23 460-479 162 162 162 277 277 277 F F F
179012024-02-07T19:02:29.373Z 24 480-499 161 161 161 276 276 276 F F F
179022024-02-07T19:02:29.373Z 25 500-519 159 159 159 273 273 273 F F F
179032024-02-07T19:02:29.373Z 26 520-539 160 160 160 275 275 275 F F F
179042024-02-07T19:02:29.373Z 27 540-559 162 162 162 277 277 277 F F F
179052024-02-07T19:02:29.373Z 28 560-579 162 162 162 277 277 277 F F F
179062024-02-07T19:02:29.373Z 29 580-599 160 160 160 274 274 274 F F F
179072024-02-07T19:02:29.373ZMax gen: 163, Max flush: 278
179082024-02-07T19:02:29.373Z
179092024-02-07T19:02:29.373ZBegin loop 82
179102024-02-07T19:02:29.373ZDownstairs to restart: 0
179112024-02-07T19:02:29.391Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594041477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791}
179122024-02-07T19:02:29.391ZCrucible runtime is spawned
179132024-02-07T19:02:29.391Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594408327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791}
179142024-02-07T19:02:29.397Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594457211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791}
179152024-02-07T19:02:29.397Z{"msg":"Crucible 5b9640b2-8f70-4da8-8d55-aaa3f53a3ccd has session id: 11d733b4-8573-4e58-a40d-93faf03aebb2","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594494186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179162024-02-07T19:02:29.397Z{"msg":"Upstairs opts: Upstairs UUID: 5b9640b2-8f70-4da8-8d55-aaa3f53a3ccd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594532541Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179172024-02-07T19:02:29.397Z{"msg":"Crucible stats registered with UUID: 5b9640b2-8f70-4da8-8d55-aaa3f53a3ccd","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594570285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179182024-02-07T19:02:29.397Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594589693Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791}
179192024-02-07T19:02:29.397Z{"msg":"5b9640b2-8f70-4da8-8d55-aaa3f53a3ccd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594627848Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179202024-02-07T19:02:29.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594658164Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179212024-02-07T19:02:29.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594707007Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179222024-02-07T19:02:29.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594769878Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179232024-02-07T19:02:29.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594810313Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179242024-02-07T19:02:29.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594843448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179252024-02-07T19:02:29.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594926247Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179262024-02-07T19:02:29.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594963522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179272024-02-07T19:02:29.397Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.594989119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179282024-02-07T19:02:29.398Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595023574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179292024-02-07T19:02:29.398Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:28.59505675Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179302024-02-07T19:02:29.398Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:28.595101654Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179312024-02-07T19:02:29.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595137429Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179322024-02-07T19:02:29.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595153437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179332024-02-07T19:02:29.398Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:28.595190472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179342024-02-07T19:02:29.398Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595216988Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179352024-02-07T19:02:29.398Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595242495Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179362024-02-07T19:02:29.398Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.59527523Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179372024-02-07T19:02:29.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595656599Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179382024-02-07T19:02:29.398Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595692634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179392024-02-07T19:02:29.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595714191Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179402024-02-07T19:02:29.398Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.595740997Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179412024-02-07T19:02:29.398Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596254888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179422024-02-07T19:02:29.398Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596376731Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179432024-02-07T19:02:29.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596771798Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179442024-02-07T19:02:29.398Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596817002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179452024-02-07T19:02:29.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596891252Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179462024-02-07T19:02:29.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596933476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179472024-02-07T19:02:29.398Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596960472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179482024-02-07T19:02:29.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:28.596989148Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179492024-02-07T19:02:30.394Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.596804403Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179502024-02-07T19:02:30.400Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.597088035Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179512024-02-07T19:02:30.400Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.598134633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179522024-02-07T19:02:30.400Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.598176078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179532024-02-07T19:02:30.400Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.598668701Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179542024-02-07T19:02:30.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599304545Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179552024-02-07T19:02:30.400Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599355088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179562024-02-07T19:02:30.400Z{"msg":"[0]R flush_numbers[0..12]: [278, 277, 278, 273, 274, 274, 277, 276, 262, 278, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599377955Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179572024-02-07T19:02:30.400Z{"msg":"[0]R generation[0..12]: [163, 162, 163, 159, 160, 160, 162, 161, 154, 163, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599407091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179582024-02-07T19:02:30.400Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,Wait for a query_work_queue command to finish before sending IO
179592024-02-07T19:02:30.400Z"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599443136Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179602024-02-07T19:02:30.400Z{"msg":"[1]R flush_numbers[0..12]: [278, 277, 278, 273, 274, 274, 277, 276, 262, 278, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599461444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179612024-02-07T19:02:30.400Z{"msg":"[1]R generation[0..12]: [163, 162, 163, 159, 160, 160, 162, 161, 154, 163, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.59948812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179622024-02-07T19:02:30.400Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599514637Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179632024-02-07T19:02:30.400Z{"msg":"[2]R flush_numbers[0..12]: [278, 277, 278, 273, 274, 274, 277, 276, 262, 278, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599533394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179642024-02-07T19:02:30.400Z{"msg":"[2]R generation[0..12]: [163, 162, 163, 159, 160, 160, 162, 161, 154, 163, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.59956118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179652024-02-07T19:02:30.400Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599580168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179662024-02-07T19:02:30.400Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599597955Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179672024-02-07T19:02:30.400Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599624292Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179682024-02-07T19:02:30.401Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.59964114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179692024-02-07T19:02:30.401Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599665836Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179702024-02-07T19:02:30.401Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599682644Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179712024-02-07T19:02:30.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599701601Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179722024-02-07T19:02:30.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599716069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179732024-02-07T19:02:30.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599730108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179742024-02-07T19:02:30.401Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599744566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179752024-02-07T19:02:30.401Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599770872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179762024-02-07T19:02:30.401Z{"msg":"5b9640b2-8f70-4da8-8d55-aaa3f53a3ccd is now active with session: 11d733b4-8573-4e58-a40d-93faf03aebb2","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.59978562Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179772024-02-07T19:02:30.401Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599798928Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791,"session_id":"11d733b4-8573-4e58-a40d-93faf03aebb2"}
179782024-02-07T19:02:30.401Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:29.599818745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791}
179792024-02-07T19:02:30.401ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179802024-02-07T19:02:30.401ZLoading write count information from file "/tmp/test_repair_verify.data"
179812024-02-07T19:02:30.401ZRun Repair workload
179822024-02-07T19:02:30.405Z01/30 Write block 122 len 28672 data: 16 17 18 18 17 15 17
179832024-02-07T19:02:30.405Z02/30 Write block 248 len 32768 data: 13 12 11 10 11 10 13 11
179842024-02-07T19:02:30.405Z03/30 Read block 385 len 12288
179852024-02-07T19:02:30.405Z04/30 Flush
179862024-02-07T19:02:30.405Z05/30 Write block 567 len 20480 data: 18 16 15 16 14
179872024-02-07T19:02:30.405Z06/30 Read block 29 len 12288
179882024-02-07T19:02:30.408Z07/30 Read block 150 len 8192
179892024-02-07T19:02:30.409Z08/30 Read block 56 len 28672
179902024-02-07T19:02:30.409Z09/30 Read block 260 len 12288
179912024-02-07T19:02:30.409Z10/30 Read block 164 len 20480
179922024-02-07T19:02:30.412Z11/30 Write block 120 len 24576 data: 15 13 17 18 19 19
179932024-02-07T19:02:30.412Z12/30 Flush
179942024-02-07T19:02:30.415Z13/30 Write block 376 len 20480 data: 14 13 13 10 11
179952024-02-07T19:02:30.415Z14/30 Write block 281 len 28672 data: 14 13 10 16 16 17 20
179962024-02-07T19:02:30.415Z15/30 Write block 464 len 28672 data: 10 11 10 7 6 8 10
179972024-02-07T19:02:30.415Z16/30 Read block 309 len 12288
179982024-02-07T19:02:30.418Z17/30 Write block 444 len 20480 data: 15 16 13 13 12
179992024-02-07T19:02:30.418Z18/30 Read block 484 len 16384
180002024-02-07T19:02:30.418Z19/30 Write block 124 len 16384 data: 20 20 18 16
180012024-02-07T19:02:30.418Z20/30 Write block 557 len 4096 data: 9
180022024-02-07T19:02:30.418Z21/30 Read block 351 len 20480
180032024-02-07T19:02:30.422Z22/30 Read block 13 len 36864
180042024-02-07T19:02:30.422Z23/30 Flush
180052024-02-07T19:02:30.427Z24/30 Write block 262 len 36864 data: 12 16 18 15 15 16 13 12 9
180062024-02-07T19:02:30.427Z25/30 Flush
180072024-02-07T19:02:30.427Z26/30 Write block 326 len 8192 data: 11 10
180082024-02-07T19:02:30.431Z27/30 Read block 507 len 32768
180092024-02-07T19:02:30.431Z28/30 Read block 314 len 20480
180102024-02-07T19:02:30.431Z29/30 Read block 175 len 8192
180112024-02-07T19:02:30.431Z30/30 Write block 283 len 40960 data: 11 17 17 18 21 19 18 20 18 15
180122024-02-07T19:02:30.431Z----------------------------------------------------------------
180132024-02-07T19:02:30.431Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:26
180142024-02-07T19:02:30.431ZStates: Active Active Active
180152024-02-07T19:02:30.431ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180162024-02-07T19:02:30.431Z 5 Acked 1004 Write 5 Sent Done Done false
180172024-02-07T19:02:30.431Z 6 Acked 1005 Read 3 Sent Done Done false
180182024-02-07T19:02:30.431Z 7 Acked 1006 Read 2 Sent Done Done false
180192024-02-07T19:02:30.431Z 8 Acked 1007 Read 7 Sent Done Done false
180202024-02-07T19:02:30.431Z 9 Acked 1008 Read 3 Sent Done Done false
180212024-02-07T19:02:30.431Z 10 Acked 1009 Read 5 Sent Done Done false
180222024-02-07T19:02:30.431Z 11 Acked 1010 Write 6 Sent Done Done false
180232024-02-07T19:02:30.431Z 12 Acked 1011 Flush 0 Sent Done Done false
180242024-02-07T19:02:30.431Z 13 Acked 1012 Write 5 Sent Done Done false
180252024-02-07T19:02:30.432Z 14 Acked 1013 Write 7 Sent Done Done false
180262024-02-07T19:02:30.432Z 15 Acked 1014 Write 7 Sent Done Done false
180272024-02-07T19:02:30.432Z 16 Acked 1015 Read 3 Sent Done Done false
180282024-02-07T19:02:30.432Z 17 Acked 1016 Write 5 Sent Done Done false
180292024-02-07T19:02:30.432Z 18 Acked 1017 Read 4 Sent Done Done false
180302024-02-07T19:02:30.432Z 19 Acked 1018 Write 4 Sent Done Done false
180312024-02-07T19:02:30.432Z 20 Acked 1019 Write 1 Sent Done Done false
180322024-02-07T19:02:30.432Z 21 Acked 1020 Read 5 Sent Done Done false
180332024-02-07T19:02:30.432Z 22 Acked 1021 Read 9 Sent Done Done false
180342024-02-07T19:02:30.432Z 23 Acked 1022 Flush 0 Sent Done Done false
180352024-02-07T19:02:30.432Z 24 Acked 1023 Write 9 Sent Done Done false
180362024-02-07T19:02:30.432Z 25 Acked 1024 Flush 0 Sent Done Done false
180372024-02-07T19:02:30.432Z 26 Acked 1025 Write 2 Sent Done Done false
180382024-02-07T19:02:30.432Z 27 Acked 1026 Read 8 Sent Done Done false
180392024-02-07T19:02:30.432Z 28 Acked 1027 Read 5 Sent Done Done false
180402024-02-07T19:02:30.432Z 29 Acked 1028 Read 2 Sent Done Done false
180412024-02-07T19:02:30.432Z 30 Acked 1029 Write 10 Sent Sent Sent false
180422024-02-07T19:02:30.432Z STATES DS:0 DS:1 DS:2 TOTAL
180432024-02-07T19:02:30.432Z New 0 0 0 0
180442024-02-07T19:02:30.432Z Sent 26 1 1 28
180452024-02-07T19:02:30.432Z Done 0 25 25 50
180462024-02-07T19:02:30.432Z Skipped 0 0 0 0
180472024-02-07T19:02:30.432Z Error 0 0 0 0
180482024-02-07T19:02:30.432ZLast Flush: 1003 1024 1024
180492024-02-07T19:02:30.432ZDownstairs last five completed: 1003 1002 1001 1000
180502024-02-07T19:02:30.433ZUpstairs last five completed: 30 29 28 27 26
180512024-02-07T19:02:30.433Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:29.632700448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1791}
180522024-02-07T19:02:30.433ZWrote out file "/tmp/test_repair_verify.data"
180532024-02-07T19:02:30.436Z
180542024-02-07T19:02:30.439ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
180552024-02-07T19:02:30.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180562024-02-07T19:02:30.455Z 0 000-019 163 163 163 278 278 278 F F F
180572024-02-07T19:02:30.455Z 1 020-039 162 162 162 277 277 277 F F F
180582024-02-07T19:02:30.455Z 2 040-059 163 163 163 278 278 278 F F F
180592024-02-07T19:02:30.455Z 3 060-079 159 159 159 273 273 273 F F F
180602024-02-07T19:02:30.455Z 4 080-099 160 160 160 274 274 274 F F F
180612024-02-07T19:02:30.455Z 5 100-119 160 160 160 274 274 274 F F F
180622024-02-07T19:02:30.459Z 6 120-139 164 164 164 279 281 281 F F F <---
180632024-02-07T19:02:30.459Z 7 140-159 161 161 161 276 276 276 F F F
180642024-02-07T19:02:30.459Z 8 160-179 154 154 154 262 262 262 F F F
180652024-02-07T19:02:30.459Z 9 180-199 163 163 163 278 278 278 F F F
180662024-02-07T19:02:30.459Z 10 200-219 160 160 160 274 274 274 F F F
180672024-02-07T19:02:30.459Z 11 220-239 155 155 155 263 263 263 F F F
180682024-02-07T19:02:30.459Z 12 240-259 164 164 164 279 279 279 F F F
180692024-02-07T19:02:30.459Z 13 260-279 161 164 164 276 282 282 F F F <---
180702024-02-07T19:02:30.459Z 14 280-299 160 164 164 274 281 281 F T T <---
180712024-02-07T19:02:30.459Z 15 300-319 163 163 163 278 278 278 F F F
180722024-02-07T19:02:30.459Z 16 320-339 163 163 163 278 278 278 F T T <---
180732024-02-07T19:02:30.460Z 17 340-359 163 163 163 278 278 278 F F F
180742024-02-07T19:02:30.460Z 18 360-379 163 164 164 278 281 281 F F F <---
180752024-02-07T19:02:30.460Z 19 380-399 158 164 164 269 281 281 F F F <---
180762024-02-07T19:02:30.460Z 20 400-419 145 145 145 245 245 245 F F F
180772024-02-07T19:02:30.460Z 21 420-439 163 163 163 278 278 278 F F F
180782024-02-07T19:02:30.460Z 22 440-459 160 164 164 274 281 281 F F F <---
180792024-02-07T19:02:30.460Z 23 460-479 162 164 164 277 281 281 F F F <---
180802024-02-07T19:02:30.460Z 24 480-499 161 161 161 276 276 276 F F F
180812024-02-07T19:02:30.460Z 25 500-519 159 159 159 273 273 273 F F F
180822024-02-07T19:02:30.460Z 26 520-539 160 160 160 275 275 275 F F F
180832024-02-07T19:02:30.460Z 27 540-559 162 164 164 277 281 281 F F F <---
180842024-02-07T19:02:30.460Z 28 560-579 162 164 164 277 280 280 F F F <---
180852024-02-07T19:02:30.460Z 29 580-599 160 160 160 274 274 274 F F F
180862024-02-07T19:02:30.460ZMax gen: 164, Max flush: 282
180872024-02-07T19:02:30.460ZError: Difference in extent metadata found!
180882024-02-07T19:02:30.460ZA Difference in extent metadata is expected here
180892024-02-07T19:02:30.460ZOn loop 82
180902024-02-07T19:02:30.460Z
180912024-02-07T19:02:30.467ZVerifying data now
180922024-02-07T19:02:34.984ZNo change after verify
180932024-02-07T19:02:34.984ZLoop: 82 Downstairs dump after verify (and repair):
180942024-02-07T19:02:34.997ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180952024-02-07T19:02:34.997Z 0 000-019 163 163 163 278 278 278 F F F
180962024-02-07T19:02:34.997Z 1 020-039 162 162 162 277 277 277 F F F
180972024-02-07T19:02:34.997Z 2 040-059 163 163 163 278 278 278 F F F
180982024-02-07T19:02:34.997Z 3 060-079 159 159 159 273 273 273 F F F
180992024-02-07T19:02:34.998Z 4 080-099 160 160 160 274 274 274 F F F
181002024-02-07T19:02:34.998Z 5 100-119 160 160 160 274 274 274 F F F
181012024-02-07T19:02:34.998Z 6 120-139 164 164 164 281 281 281 F F F
181022024-02-07T19:02:34.998Z 7 140-159 161 161 161 276 276 276 F F F
181032024-02-07T19:02:34.998Z 8 160-179 154 154 154 262 262 262 F F F
181042024-02-07T19:02:34.998Z 9 180-199 163 163 163 278 278 278 F F F
181052024-02-07T19:02:35.004Z 10 200-219 160 160 160 274 274 274 F F F
181062024-02-07T19:02:35.004Z 11 220-239 155 155 155 263 263 263 F F F
181072024-02-07T19:02:35.004Z 12 240-259 164 164 164 279 279 279 F F F
181082024-02-07T19:02:35.005Z 13 260-279 164 164 164 282 282 282 F F F
181092024-02-07T19:02:35.005Z 14 280-299 165 165 165 283 283 283 F F F
181102024-02-07T19:02:35.005Z 15 300-319 163 163 163 278 278 278 F F F
181112024-02-07T19:02:35.005Z 16 320-339 165 165 165 283 283 283 F F F
181122024-02-07T19:02:35.005Z 17 340-359 163 163 163 278 278 278 F F F
181132024-02-07T19:02:35.005Z 18 360-379 164 164 164 281 281 281 F F F
181142024-02-07T19:02:35.005Z 19 380-399 164 164 164 281 281 281 F F F
181152024-02-07T19:02:35.005Z 20 400-419 145 145 145 245 245 245 F F F
181162024-02-07T19:02:35.005Z 21 420-439 163 163 163 278 278 278 F F F
181172024-02-07T19:02:35.005Z 22 440-459 164 164 164 281 281 281 F F F
181182024-02-07T19:02:35.005Z 23 460-479 164 164 164 281 281 281 F F F
181192024-02-07T19:02:35.005Z 24 480-499 161 161 161 276 276 276 F F F
181202024-02-07T19:02:35.005Z 25 500-519 159 159 159 273 273 273 F F F
181212024-02-07T19:02:35.006Z 26 520-539 160 160 160 275 275 275 F F F
181222024-02-07T19:02:35.006Z 27 540-559 164 164 164 281 281 281 F F F
181232024-02-07T19:02:35.006Z 28 560-579 164 164 164 280 280 280 F F F
181242024-02-07T19:02:35.006Z 29 580-599 160 160 160 274 274 274 F F F
181252024-02-07T19:02:35.006ZMax gen: 165, Max flush: 283
181262024-02-07T19:02:35.006Z
181272024-02-07T19:02:35.006ZBegin loop 83
181282024-02-07T19:02:35.006ZDownstairs to restart: 2
181292024-02-07T19:02:35.018Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220171111Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799}
181302024-02-07T19:02:35.018Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220436195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799}
181312024-02-07T19:02:35.024Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.22047535Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799}
181322024-02-07T19:02:35.024Z{Crucible runtime is spawned
181332024-02-07T19:02:35.024Z"msg":"Crucible 2c3ae588-d0f7-422f-899b-fe61c5acacf9 has session id: 66a124cf-7a1b-47d0-870f-9fc711738553","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220521583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181342024-02-07T19:02:35.024Z{"msg":"Upstairs opts: Upstairs UUID: 2c3ae588-d0f7-422f-899b-fe61c5acacf9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220582735Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181352024-02-07T19:02:35.024Z{"msg":"Crucible stats registered with UUID: 2c3ae588-d0f7-422f-899b-fe61c5acacf9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220638757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181362024-02-07T19:02:35.024Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220675083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799}
181372024-02-07T19:02:35.024Z{"msg":"2c3ae588-d0f7-422f-899b-fe61c5acacf9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220828472Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181382024-02-07T19:02:35.024Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220882724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181392024-02-07T19:02:35.024Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220941836Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181402024-02-07T19:02:35.025Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.220981871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181412024-02-07T19:02:35.025Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221009057Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181422024-02-07T19:02:35.025Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221036064Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181432024-02-07T19:02:35.025Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.22106368Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181442024-02-07T19:02:35.025Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221092266Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181452024-02-07T19:02:35.025Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221132421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181462024-02-07T19:02:35.025Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221169716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181472024-02-07T19:02:35.025Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221207791Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181482024-02-07T19:02:35.025Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221235987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181492024-02-07T19:02:35.025Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221273202Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181502024-02-07T19:02:35.025Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221839275Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181512024-02-07T19:02:35.025Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.22187643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181522024-02-07T19:02:35.025Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221892698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181532024-02-07T19:02:35.025Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.221914905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181542024-02-07T19:02:35.025Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.222523093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181552024-02-07T19:02:35.025Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.222672082Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181562024-02-07T19:02:35.025Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223162296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181572024-02-07T19:02:35.025Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223200861Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181582024-02-07T19:02:35.025Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223217359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181592024-02-07T19:02:35.025Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223247764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181602024-02-07T19:02:35.026Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223269112Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181612024-02-07T19:02:35.026Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223361349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181622024-02-07T19:02:35.026Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223390865Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181632024-02-07T19:02:35.026Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223408853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181642024-02-07T19:02:35.026Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.223804409Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181652024-02-07T19:02:35.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224288874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181662024-02-07T19:02:35.038Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224328628Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181672024-02-07T19:02:35.038Z{"msg":"[0]R flush_numbers[0..12]: [278, 277, 278, 273, 274, 274, 281, 276, 262, 278, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224345146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181682024-02-07T19:02:35.038Z{"msg":"[0]R generation[0..12]: [163, 162, 163, 159, 160, 160, 164, 161, 154, 163, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224367163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181692024-02-07T19:02:35.038Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224382471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181702024-02-07T19:02:35.038Z{"msg":"[1]R flush_numbers[0..12]: [278, 277, 278, 273, 274, 274, 281, 276, 262, 278, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224404048Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181712024-02-07T19:02:35.038Z{"msg":"[1]R generation[0..12]: [163, 162, 163, 159, 160, 160, 164, 161, 154, 163, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224418386Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181722024-02-07T19:02:35.038Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224439234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181732024-02-07T19:02:35.038Z{"msg":"[2]R flush_numbers[0..12]: [278, 277, 278, 273, 274, 274, 281, 276, 262, 278, 274, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224454961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181742024-02-07T19:02:35.038ZWait for a query_work_queue command to finish before sending IO
181752024-02-07T19:02:35.038Z{"msg":"[2]R generation[0..12]: [163, 162, 163, 159, 160, 160, 164, 161, 154, 163, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224481408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181762024-02-07T19:02:35.038Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224496866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181772024-02-07T19:02:35.039Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224517123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181782024-02-07T19:02:35.039Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224531371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181792024-02-07T19:02:35.039Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224551338Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181802024-02-07T19:02:35.039Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224565296Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181812024-02-07T19:02:35.039Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224586054Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181822024-02-07T19:02:35.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224598722Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181832024-02-07T19:02:35.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224635917Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181842024-02-07T19:02:35.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224650445Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181852024-02-07T19:02:35.039Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224664143Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181862024-02-07T19:02:35.039Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.22468377Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181872024-02-07T19:02:35.039Z{"msg":"2c3ae588-d0f7-422f-899b-fe61c5acacf9 is now active with session: 66a124cf-7a1b-47d0-870f-9fc711738553","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224697188Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181882024-02-07T19:02:35.039Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224716716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799,"session_id":"66a124cf-7a1b-47d0-870f-9fc711738553"}
181892024-02-07T19:02:35.040Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:34.224729574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799}
181902024-02-07T19:02:35.040ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181912024-02-07T19:02:35.040ZLoading write count information from file "/tmp/test_repair_verify.data"
181922024-02-07T19:02:35.040ZRun Repair workload
181932024-02-07T19:02:35.040Z01/30 Write block 6 len 4096 data: 14
181942024-02-07T19:02:35.040Z02/30 Read block 77 len 8192
181952024-02-07T19:02:35.040Z03/30 Write block 471 len 8192 data: 9 9
181962024-02-07T19:02:35.040Z04/30 Read block 103 len 32768
181972024-02-07T19:02:35.040Z05/30 Write block 228 len 40960 data: 10 12 11 10 14 14 14 21 20 21
181982024-02-07T19:02:35.040Z06/30 Read block 559 len 4096
181992024-02-07T19:02:35.040Z07/30 Flush
182002024-02-07T19:02:35.054Z08/30 Write block 553 len 16384 data: 12 11 13 12
182012024-02-07T19:02:35.054Z09/30 Write block 37 len 32768 data: 12 11 9 10 10 9 9 9
182022024-02-07T19:02:35.054Z10/30 Write block 221 len 8192 data: 9 9
182032024-02-07T19:02:35.054Z11/30 Write block 597 len 8192 data: 6 4
182042024-02-07T19:02:35.054Z12/30 Read block 379 len 4096
182052024-02-07T19:02:35.055Z13/30 Write block 299 len 20480 data: 9 8 9 11 7
182062024-02-07T19:02:35.055Z14/30 Read block 467 len 32768
182072024-02-07T19:02:35.055Z15/30 Read block 109 len 16384
182082024-02-07T19:02:35.055Z16/30 Write block 68 len 12288 data: 15 14 13
182092024-02-07T19:02:35.055Z17/30 Write block 454 len 32768 data: 11 10 8 12 14 13 12 11
182102024-02-07T19:02:35.055Z18/30 Write block 50 len 8192 data: 9 12
182112024-02-07T19:02:35.055Z19/30 Flush
182122024-02-07T19:02:35.055Z20/30 Write block 251 len 16384 data: 11 12 11 14
182132024-02-07T19:02:35.055Z21/30 Read block 128 len 16384
182142024-02-07T19:02:35.063Z22/30 Write block 22 len 16384 data: 17 20 17 16
182152024-02-07T19:02:35.063Z23/30 Read block 190 len 32768
182162024-02-07T19:02:35.063Z24/30 Read block 416 len 32768
182172024-02-07T19:02:35.063Z25/30 Read block 134 len 40960
182182024-02-07T19:02:35.063Z26/30 Read block 182 len 12288
182192024-02-07T19:02:35.063Z27/30 Write block 333 len 24576 data: 15 15 14 14 13 15
182202024-02-07T19:02:35.063Z28/30 Read block 47 len 4096
182212024-02-07T19:02:35.063Z29/30 Read block 181 len 32768
182222024-02-07T19:02:35.063Z30/30 Write block 422 len 8192 data: 8 9
182232024-02-07T19:02:35.063Z----------------------------------------------------------------
182242024-02-07T19:02:35.063Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
182252024-02-07T19:02:35.063ZStates: Active Active Active
182262024-02-07T19:02:35.064ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182272024-02-07T19:02:35.064Z 1 Acked 1000 Write 1 Done Done Done false
182282024-02-07T19:02:35.064Z 2 Acked 1001 Read 2 Done Done Done false
182292024-02-07T19:02:35.064Z 3 Acked 1002 Write 2 Done Done Done false
182302024-02-07T19:02:35.064Z 4 Acked 1003 Read 8 Done Done Done false
182312024-02-07T19:02:35.064Z 5 Acked 1004 Write 10 Done Done Done false
182322024-02-07T19:02:35.064Z 6 Acked 1005 Read 1 Done Done Done false
182332024-02-07T19:02:35.064Z 7 Acked 1006 Flush 0 Done Done Sent false
182342024-02-07T19:02:35.064Z 8 Acked 1007 Write 4 Done Done Sent false
182352024-02-07T19:02:35.064Z 9 Acked 1008 Write 8 Done Done Sent false
182362024-02-07T19:02:35.064Z 10 Acked 1009 Write 2 Done Done Sent false
182372024-02-07T19:02:35.064Z 11 Acked 1010 Write 2 Done Done Sent false
182382024-02-07T19:02:35.064Z 12 Acked 1011 Read 1 Done Done Sent false
182392024-02-07T19:02:35.065Z 13 Acked 1012 Write 5 Done Done Sent false
182402024-02-07T19:02:35.065Z 14 Acked 1013 Read 8 Done Done Sent false
182412024-02-07T19:02:35.065Z 15 Acked 1014 Read 4 Done Done Sent false
182422024-02-07T19:02:35.065Z 16 Acked 1015 Write 3 Done Done Sent false
182432024-02-07T19:02:35.065Z 17 Acked 1016 Write 8 Done Done Sent false
182442024-02-07T19:02:35.065Z 18 Acked 1017 Write 2 Done Done Sent false
182452024-02-07T19:02:35.065Z 19 Acked 1018 Flush 0 Done Done Sent false
182462024-02-07T19:02:35.065Z 20 Acked 1019 Write 4 Done Done Sent false
182472024-02-07T19:02:35.065Z 21 Acked 1020 Read 4 Done Done Sent false
182482024-02-07T19:02:35.065Z 22 Acked 1021 Write 4 Done Done Sent false
182492024-02-07T19:02:35.065Z 23 Acked 1022 Read 8 Done Done Sent false
182502024-02-07T19:02:35.065Z 24 Acked 1023 Read 8 Done Done Sent false
182512024-02-07T19:02:35.065Z 25 Acked 1024 Read 10 Done Done Sent false
182522024-02-07T19:02:35.065Z 26 Acked 1025 Read 3 Done Done Sent false
182532024-02-07T19:02:35.066Z 27 Acked 1026 Write 6 Done Done Sent false
182542024-02-07T19:02:35.066Z 28 Acked 1027 Read 1 Done Done Sent false
182552024-02-07T19:02:35.066Z 29 Acked 1028 Read 8 Done Done Sent false
182562024-02-07T19:02:35.066Z 30 Acked 1029 Write 2 Sent Sent Sent false
182572024-02-07T19:02:35.066Z STATES DS:0 DS:1 DS:2 TOTAL
182582024-02-07T19:02:35.066Z New 0 0 0 0
182592024-02-07T19:02:35.066Z Sent 1 1 24 26
182602024-02-07T19:02:35.066Z Done 29 29 6 64
182612024-02-07T19:02:35.066Z Skipped 0 0 0 0
182622024-02-07T19:02:35.066Z Error 0 0 0 0
182632024-02-07T19:02:35.066ZLast Flush: 1018 1018 0
182642024-02-07T19:02:35.066ZDownstairs last five completed:
182652024-02-07T19:02:35.066ZUpstairs last five completed: 30 29 28 27 26
182662024-02-07T19:02:35.067Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:34.254922301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1799}
182672024-02-07T19:02:35.067ZWrote out file "/tmp/test_repair_verify.data"
182682024-02-07T19:02:35.067Z
182692024-02-07T19:02:35.081ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182702024-02-07T19:02:35.087ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182712024-02-07T19:02:35.087Z 0 000-019 166 166 163 284 284 278 F F T <---
182722024-02-07T19:02:35.087Z 1 020-039 166 166 162 285 285 277 T T F <---
182732024-02-07T19:02:35.087Z 2 040-059 166 166 163 285 285 278 F F F <---
182742024-02-07T19:02:35.087Z 3 060-079 166 166 159 285 285 273 F F F <---
182752024-02-07T19:02:35.087Z 4 080-099 160 160 160 274 274 274 F F F
182762024-02-07T19:02:35.087Z 5 100-119 160 160 160 274 274 274 F F F
182772024-02-07T19:02:35.087Z 6 120-139 164 164 164 281 281 281 F F F
182782024-02-07T19:02:35.087Z 7 140-159 161 161 161 276 276 276 F F F
182792024-02-07T19:02:35.087Z 8 160-179 154 154 154 262 262 262 F F F
182802024-02-07T19:02:35.087Z 9 180-199 163 163 163 278 278 278 F F F
182812024-02-07T19:02:35.087Z 10 200-219 160 160 160 274 274 274 F F F
182822024-02-07T19:02:35.087Z 11 220-239 166 166 155 285 285 263 F F T <---
182832024-02-07T19:02:35.087Z 12 240-259 164 164 164 279 279 279 T T F <---
182842024-02-07T19:02:35.087Z 13 260-279 164 164 164 282 282 282 F F F
182852024-02-07T19:02:35.088Z 14 280-299 166 166 165 285 285 283 F F F <---
182862024-02-07T19:02:35.088Z 15 300-319 166 166 163 285 285 278 F F F <---
182872024-02-07T19:02:35.088Z 16 320-339 165 165 165 283 283 283 T T F <---
182882024-02-07T19:02:35.088Z 17 340-359 163 163 163 278 278 278 F F F
182892024-02-07T19:02:35.088Z 18 360-379 164 164 164 281 281 281 F F F
182902024-02-07T19:02:35.088Z 19 380-399 164 164 164 281 281 281 F F F
182912024-02-07T19:02:35.088Z 20 400-419 145 145 145 245 245 245 F F F
182922024-02-07T19:02:35.088Z 21 420-439 163 163 163 278 278 278 T T F <---
182932024-02-07T19:02:35.088Z 22 440-459 166 166 164 285 285 281 F F F <---
182942024-02-07T19:02:35.088Z 23 460-479 166 166 164 285 285 281 F F T <---
182952024-02-07T19:02:35.088Z 24 480-499 161 161 161 276 276 276 F F F
182962024-02-07T19:02:35.088Z 25 500-519 159 159 159 273 273 273 F F F
182972024-02-07T19:02:35.088Z 26 520-539 160 160 160 275 275 275 F F F
182982024-02-07T19:02:35.088Z 27 540-559 166 166 164 285 285 281 F F F <---
182992024-02-07T19:02:35.088Z 28 560-579 164 164 164 280 280 280 F F F
183002024-02-07T19:02:35.088Z 29 580-599 166 166 160 285 285 274 F F F <---
183012024-02-07T19:02:35.088ZMax gen: 166, Max flush: 285
183022024-02-07T19:02:35.088ZError: Difference in extent metadata found!
183032024-02-07T19:02:35.088ZA Difference in extent metadata is expected here
183042024-02-07T19:02:35.088ZOn loop 83
183052024-02-07T19:02:35.089Z
183062024-02-07T19:02:35.089ZVerifying data now
183072024-02-07T19:02:40.767ZNo change after verify
183082024-02-07T19:02:40.767ZLoop: 83 Downstairs dump after verify (and repair):
183092024-02-07T19:02:40.782ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183102024-02-07T19:02:40.782Z 0 000-019 166 166 166 284 284 284 F F F
183112024-02-07T19:02:40.782Z 1 020-039 167 167 167 286 286 286 F F F
183122024-02-07T19:02:40.782Z 2 040-059 166 166 166 285 285 285 F F F
183132024-02-07T19:02:40.783Z 3 060-079 166 166 166 285 285 285 F F F
183142024-02-07T19:02:40.783Z 4 080-099 160 160 160 274 274 274 F F F
183152024-02-07T19:02:40.783Z 5 100-119 160 160 160 274 274 274 F F F
183162024-02-07T19:02:40.783Z 6 120-139 164 164 164 281 281 281 F F F
183172024-02-07T19:02:40.783Z 7 140-159 161 161 161 276 276 276 F F F
183182024-02-07T19:02:40.783Z 8 160-179 154 154 154 262 262 262 F F F
183192024-02-07T19:02:40.783Z 9 180-199 163 163 163 278 278 278 F F F
183202024-02-07T19:02:40.783Z 10 200-219 160 160 160 274 274 274 F F F
183212024-02-07T19:02:40.783Z 11 220-239 166 166 166 285 285 285 F F F
183222024-02-07T19:02:40.786Z 12 240-259 167 167 167 286 286 286 F F F
183232024-02-07T19:02:40.786Z 13 260-279 164 164 164 282 282 282 F F F
183242024-02-07T19:02:40.786Z 14 280-299 166 166 166 285 285 285 F F F
183252024-02-07T19:02:40.786Z 15 300-319 166 166 166 285 285 285 F F F
183262024-02-07T19:02:40.786Z 16 320-339 167 167 167 286 286 286 F F F
183272024-02-07T19:02:40.786Z 17 340-359 163 163 163 278 278 278 F F F
183282024-02-07T19:02:40.786Z 18 360-379 164 164 164 281 281 281 F F F
183292024-02-07T19:02:40.786Z 19 380-399 164 164 164 281 281 281 F F F
183302024-02-07T19:02:40.786Z 20 400-419 145 145 145 245 245 245 F F F
183312024-02-07T19:02:40.787Z 21 420-439 167 167 167 286 286 286 F F F
183322024-02-07T19:02:40.787Z 22 440-459 166 166 166 285 285 285 F F F
183332024-02-07T19:02:40.787Z 23 460-479 166 166 166 285 285 285 F F F
183342024-02-07T19:02:40.787Z 24 480-499 161 161 161 276 276 276 F F F
183352024-02-07T19:02:40.787Z 25 500-519 159 159 159 273 273 273 F F F
183362024-02-07T19:02:40.787Z 26 520-539 160 160 160 275 275 275 F F F
183372024-02-07T19:02:40.787Z 27 540-559 166 166 166 285 285 285 F F F
183382024-02-07T19:02:40.787Z 28 560-579 164 164 164 280 280 280 F F F
183392024-02-07T19:02:40.787Z 29 580-599 166 166 166 285 285 285 F F F
183402024-02-07T19:02:40.787ZMax gen: 167, Max flush: 286
183412024-02-07T19:02:40.787Z
183422024-02-07T19:02:40.787ZBegin loop 84
183432024-02-07T19:02:40.787ZDownstairs to restart: 1
183442024-02-07T19:02:40.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.00535681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807}
183452024-02-07T19:02:40.805Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005592089Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807}
183462024-02-07T19:02:40.805Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005617905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807}
183472024-02-07T19:02:40.812Z{"msg":"Crucible 24c92344-6ea6-4343-a114-6d13e881b5c9 has session id: 1611449a-5726-497e-8fa3-275f132805a0","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005653561Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183482024-02-07T19:02:40.812Z{"msg":"Crucible runtime is spawned
183492024-02-07T19:02:40.812ZUpstairs opts: Upstairs UUID: 24c92344-6ea6-4343-a114-6d13e881b5c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005705163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183502024-02-07T19:02:40.812Z{"msg":"Crucible stats registered with UUID: 24c92344-6ea6-4343-a114-6d13e881b5c9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005739009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183512024-02-07T19:02:40.812Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005763515Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807}
183522024-02-07T19:02:40.812Z{"msg":"24c92344-6ea6-4343-a114-6d13e881b5c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005883859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183532024-02-07T19:02:40.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005915945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183542024-02-07T19:02:40.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.00595295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183552024-02-07T19:02:40.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.005994244Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183562024-02-07T19:02:40.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.00602244Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183572024-02-07T19:02:40.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006057576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183582024-02-07T19:02:40.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183592024-02-07T19:02:40.813Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006139525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183602024-02-07T19:02:40.813Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.0061757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183612024-02-07T19:02:40.813Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006201566Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183622024-02-07T19:02:40.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006238901Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183632024-02-07T19:02:40.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006268697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183642024-02-07T19:02:40.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.006306462Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183652024-02-07T19:02:40.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.007052471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183662024-02-07T19:02:40.813Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.007100625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183672024-02-07T19:02:40.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.007141569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183682024-02-07T19:02:40.813Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.007180604Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183692024-02-07T19:02:40.813Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.007844374Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183702024-02-07T19:02:40.813Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.007881819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183712024-02-07T19:02:40.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008209635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183722024-02-07T19:02:40.813Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008261388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183732024-02-07T19:02:40.814Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008487267Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183742024-02-07T19:02:40.814Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008529301Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183752024-02-07T19:02:40.814Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008553768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183762024-02-07T19:02:40.814Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008592723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183772024-02-07T19:02:40.814Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008619419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183782024-02-07T19:02:40.814Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008649795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183792024-02-07T19:02:40.814Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.008812053Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183802024-02-07T19:02:40.814Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009288649Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183812024-02-07T19:02:40.814Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009328483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183822024-02-07T19:02:40.814Z{"msg":"[0]R flush_numbers[0..12]: [284, 286, 285, 285, 274, 274, 281, 276, 262, 278, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009346311Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183832024-02-07T19:02:40.814Z{"msg":"[0]R generation[0..12]: [166, 167, 166, 166, 160, 160, 164, 161, 154, 163, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009369148Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183842024-02-07T19:02:40.814Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009384906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183852024-02-07T19:02:40.814Z{"msg":"[1]R flush_numbers[0..12]: [284, 286, 285, 285, 274, 274, 281, 276, 262, 278, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009406293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183862024-02-07T19:02:40.814Z{"msg":"[1]R generation[0..12]: [166, 167, 166, 166, 160, 160, 164, 161, 154, 163, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009421031Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183872024-02-07T19:02:40.814Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009442518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183882024-02-07T19:02:40.815Z{"msg":"[2]R flush_numbers[0..12]: [284, 286, 285, 285, 274, 274, 281, 276, 262, 278, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009457516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183892024-02-07T19:02:40.815ZWait for a query_work_queue command to finish before sending IO
183902024-02-07T19:02:40.815Z{"msg":"[2]R generation[0..12]: [166, 167, 166, 166, 160, 160, 164, 161, 154, 163, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009483363Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183912024-02-07T19:02:40.815Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.00949959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183922024-02-07T19:02:40.815Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009514918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183932024-02-07T19:02:40.815Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009530156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183942024-02-07T19:02:40.815Z{"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009550443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183952024-02-07T19:02:40.815Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009564392Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183962024-02-07T19:02:40.815Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009584899Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183972024-02-07T19:02:40.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009597677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183982024-02-07T19:02:40.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009611075Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
183992024-02-07T19:02:40.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009631012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
184002024-02-07T19:02:40.815Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009646201Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
184012024-02-07T19:02:40.815Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009659009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
184022024-02-07T19:02:40.815Z{"msg":"24c92344-6ea6-4343-a114-6d13e881b5c9 is now active with session: 1611449a-5726-497e-8fa3-275f132805a0","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009672387Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
184032024-02-07T19:02:40.815Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009684635Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807,"session_id":"1611449a-5726-497e-8fa3-275f132805a0"}
184042024-02-07T19:02:40.816Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:40.009697334Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807}
184052024-02-07T19:02:40.820ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184062024-02-07T19:02:40.820ZLoading write count information from file "/tmp/test_repair_verify.data"
184072024-02-07T19:02:40.820ZRun Repair workload
184082024-02-07T19:02:40.820Z01/30 Write block 453 len 20480 data: 12 12 11 9 13
184092024-02-07T19:02:40.820Z02/30 Write block 101 len 8192 data: 10 10
184102024-02-07T19:02:40.820Z03/30 Write block 2 len 20480 data: 4 6 9 13 15
184112024-02-07T19:02:40.820Z04/30 Read block 482 len 12288
184122024-02-07T19:02:40.821Z05/30 Flush
184132024-02-07T19:02:40.821Z06/30 Write block 428 len 8192 data: 16 14
184142024-02-07T19:02:40.821Z07/30 Write block 166 len 8192 data: 14 13
184152024-02-07T19:02:40.821Z08/30 Write block 580 len 28672 data: 12 13 15 13 14 15 14
184162024-02-07T19:02:40.821Z09/30 Read block 442 len 24576
184172024-02-07T19:02:40.821Z10/30 Write block 555 len 24576 data: 14 13 10 11 13 14
184182024-02-07T19:02:40.821Z11/30 Read block 488 len 32768
184192024-02-07T19:02:40.824Z12/30 Write block 172 len 40960 data: 13 12 10 10 9 10 10 10 12 7
184202024-02-07T19:02:40.825Z13/30 Flush
184212024-02-07T19:02:40.828Z14/30 Flush
184222024-02-07T19:02:40.828Z15/30 Write block 572 len 24576 data: 12 12 9 10 10 10
184232024-02-07T19:02:40.828Z16/30 Flush
184242024-02-07T19:02:40.831Z17/30 Write block 545 len 32768 data: 8 10 15 17 18 17 14 12
184252024-02-07T19:02:40.831Z18/30 Read block 316 len 36864
184262024-02-07T19:02:40.834Z19/30 Flush
184272024-02-07T19:02:40.838Z20/30 Flush
184282024-02-07T19:02:40.838Z21/30 Write block 293 len 8192 data: 14 12
184292024-02-07T19:02:40.838Z22/30 Write block 518 len 16384 data: 15 13 13 11
184302024-02-07T19:02:40.838Z23/30 Write block 333 len 32768 data: 16 16 15 15 14 16 19 15
184312024-02-07T19:02:40.838Z24/30 Read block 135 len 8192
184322024-02-07T19:02:40.841Z25/30 Read block 438 len 32768
184332024-02-07T19:02:40.842Z26/30 Read block 398 len 40960
184342024-02-07T19:02:40.842Z27/30 Write block 75 len 8192 data: 8 13
184352024-02-07T19:02:40.842Z28/30 Read block 174 len 8192
184362024-02-07T19:02:40.842Z29/30 Read block 170 len 40960
184372024-02-07T19:02:40.846Z30/30 Write block 540 len 8192 data: 14 15
184382024-02-07T19:02:40.846Z----------------------------------------------------------------
184392024-02-07T19:02:40.846Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:16
184402024-02-07T19:02:40.846ZStates: Active Active Active
184412024-02-07T19:02:40.846ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184422024-02-07T19:02:40.846Z 15 Acked 1014 Write 6 Done Done Done false
184432024-02-07T19:02:40.846Z 16 Acked 1015 Flush 0 Done Sent Done false
184442024-02-07T19:02:40.847Z 17 Acked 1016 Write 8 Done Sent Done false
184452024-02-07T19:02:40.847Z 18 Acked 1017 Read 9 Done Sent Done false
184462024-02-07T19:02:40.847Z 19 Acked 1018 Flush 0 Done Sent Done false
184472024-02-07T19:02:40.847Z 20 Acked 1019 Flush 0 Done Sent Done false
184482024-02-07T19:02:40.847Z 21 Acked 1020 Write 2 Done Sent Done false
184492024-02-07T19:02:40.847Z 22 Acked 1021 Write 4 Done Sent Done false
184502024-02-07T19:02:40.847Z 23 Acked 1022 Write 8 Done Sent Done false
184512024-02-07T19:02:40.847Z 24 Acked 1023 Read 2 Done Sent Done false
184522024-02-07T19:02:40.847Z 25 Acked 1024 Read 8 Done Sent Done false
184532024-02-07T19:02:40.847Z 26 Acked 1025 Read 10 Done Sent Done false
184542024-02-07T19:02:40.847Z 27 Acked 1026 Write 2 Done Sent Done false
184552024-02-07T19:02:40.847Z 28 Acked 1027 Read 2 Done Sent Done false
184562024-02-07T19:02:40.847Z 29 Acked 1028 Read 10 Done Sent Done false
184572024-02-07T19:02:40.848Z 30 Acked 1029 Write 2 Sent Sent Sent false
184582024-02-07T19:02:40.848Z STATES DS:0 DS:1 DS:2 TOTAL
184592024-02-07T19:02:40.848Z New 0 0 0 0
184602024-02-07T19:02:40.848Z Sent 1 15 1 17
184612024-02-07T19:02:40.848Z Done 15 1 15 31
184622024-02-07T19:02:40.848Z Skipped 0 0 0 0
184632024-02-07T19:02:40.848Z Error 0 0 0 0
184642024-02-07T19:02:40.848ZLast Flush: 1019 1013 1019
184652024-02-07T19:02:40.848ZDownstairs last five completed: 1013 1012 1011 1010 1009
184662024-02-07T19:02:40.848ZUpstairs last five completed: 30 29 28 27 26
184672024-02-07T19:02:40.848Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:40.04462497Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1807}
184682024-02-07T19:02:40.848ZWrote out file "/tmp/test_repair_verify.data"
184692024-02-07T19:02:40.852Z
184702024-02-07T19:02:40.852ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
184712024-02-07T19:02:40.869ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184722024-02-07T19:02:40.869Z 0 000-019 168 168 168 287 287 287 F F F
184732024-02-07T19:02:40.869Z 1 020-039 167 167 167 286 286 286 F F F
184742024-02-07T19:02:40.869Z 2 040-059 166 166 166 285 285 285 F F F
184752024-02-07T19:02:40.869Z 3 060-079 166 166 166 285 285 285 T F T <---
184762024-02-07T19:02:40.870Z 4 080-099 160 160 160 274 274 274 F F F
184772024-02-07T19:02:40.870Z 5 100-119 168 168 168 287 287 287 F F F
184782024-02-07T19:02:40.870Z 6 120-139 164 164 164 281 281 281 F F F
184792024-02-07T19:02:40.870Z 7 140-159 161 161 161 276 276 276 F F F
184802024-02-07T19:02:40.870Z 8 160-179 168 168 168 288 288 288 F F F
184812024-02-07T19:02:40.870Z 9 180-199 168 168 168 288 288 288 F F F
184822024-02-07T19:02:40.870Z 10 200-219 160 160 160 274 274 274 F F F
184832024-02-07T19:02:40.870Z 11 220-239 166 166 166 285 285 285 F F F
184842024-02-07T19:02:40.870Z 12 240-259 167 167 167 286 286 286 F F F
184852024-02-07T19:02:40.870Z 13 260-279 164 164 164 282 282 282 F F F
184862024-02-07T19:02:40.874Z 14 280-299 166 166 166 285 285 285 T F T <---
184872024-02-07T19:02:40.874Z 15 300-319 166 166 166 285 285 285 F F F
184882024-02-07T19:02:40.874Z 16 320-339 167 167 167 286 286 286 T F T <---
184892024-02-07T19:02:40.874Z 17 340-359 163 163 163 278 278 278 T F T <---
184902024-02-07T19:02:40.874Z 18 360-379 164 164 164 281 281 281 F F F
184912024-02-07T19:02:40.874Z 19 380-399 164 164 164 281 281 281 F F F
184922024-02-07T19:02:40.874Z 20 400-419 145 145 145 245 245 245 F F F
184932024-02-07T19:02:40.874Z 21 420-439 168 168 168 288 288 288 F F F
184942024-02-07T19:02:40.874Z 22 440-459 168 168 168 287 287 287 F F F
184952024-02-07T19:02:40.874Z 23 460-479 166 166 166 285 285 285 F F F
184962024-02-07T19:02:40.874Z 24 480-499 161 161 161 276 276 276 F F F
184972024-02-07T19:02:40.874Z 25 500-519 159 159 159 273 273 273 T F T <---
184982024-02-07T19:02:40.874Z 26 520-539 160 160 160 275 275 275 T F T <---
184992024-02-07T19:02:40.874Z 27 540-559 168 168 168 291 288 291 T F T <---
185002024-02-07T19:02:40.875Z 28 560-579 168 168 168 290 288 290 F T F <---
185012024-02-07T19:02:40.875Z 29 580-599 168 168 168 288 288 288 F F F
185022024-02-07T19:02:40.875ZMax gen: 168, Max flush: 291
185032024-02-07T19:02:40.875ZError: Difference in extent metadata found!
185042024-02-07T19:02:40.875ZA Difference in extent metadata is expected here
185052024-02-07T19:02:40.875ZOn loop 84
185062024-02-07T19:02:40.875Z
185072024-02-07T19:02:40.879ZVerifying data now
185082024-02-07T19:02:46.332ZNo change after verify
185092024-02-07T19:02:46.332ZLoop: 84 Downstairs dump after verify (and repair):
185102024-02-07T19:02:46.347ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185112024-02-07T19:02:46.347Z 0 000-019 168 168 168 287 287 287 F F F
185122024-02-07T19:02:46.347Z 1 020-039 167 167 167 286 286 286 F F F
185132024-02-07T19:02:46.347Z 2 040-059 166 166 166 285 285 285 F F F
185142024-02-07T19:02:46.347Z 3 060-079 169 169 169 292 292 292 F F F
185152024-02-07T19:02:46.347Z 4 080-099 160 160 160 274 274 274 F F F
185162024-02-07T19:02:46.347Z 5 100-119 168 168 168 287 287 287 F F F
185172024-02-07T19:02:46.347Z 6 120-139 164 164 164 281 281 281 F F F
185182024-02-07T19:02:46.347Z 7 140-159 161 161 161 276 276 276 F F F
185192024-02-07T19:02:46.347Z 8 160-179 168 168 168 288 288 288 F F F
185202024-02-07T19:02:46.347Z 9 180-199 168 168 168 288 288 288 F F F
185212024-02-07T19:02:46.351Z 10 200-219 160 160 160 274 274 274 F F F
185222024-02-07T19:02:46.351Z 11 220-239 166 166 166 285 285 285 F F F
185232024-02-07T19:02:46.351Z 12 240-259 167 167 167 286 286 286 F F F
185242024-02-07T19:02:46.351Z 13 260-279 164 164 164 282 282 282 F F F
185252024-02-07T19:02:46.351Z 14 280-299 169 169 169 292 292 292 F F F
185262024-02-07T19:02:46.351Z 15 300-319 166 166 166 285 285 285 F F F
185272024-02-07T19:02:46.351Z 16 320-339 169 169 169 292 292 292 F F F
185282024-02-07T19:02:46.351Z 17 340-359 169 169 169 292 292 292 F F F
185292024-02-07T19:02:46.351Z 18 360-379 164 164 164 281 281 281 F F F
185302024-02-07T19:02:46.351Z 19 380-399 164 164 164 281 281 281 F F F
185312024-02-07T19:02:46.351Z 20 400-419 145 145 145 245 245 245 F F F
185322024-02-07T19:02:46.351Z 21 420-439 168 168 168 288 288 288 F F F
185332024-02-07T19:02:46.351Z 22 440-459 168 168 168 287 287 287 F F F
185342024-02-07T19:02:46.351Z 23 460-479 166 166 166 285 285 285 F F F
185352024-02-07T19:02:46.351Z 24 480-499 161 161 161 276 276 276 F F F
185362024-02-07T19:02:46.352Z 25 500-519 169 169 169 292 292 292 F F F
185372024-02-07T19:02:46.352Z 26 520-539 169 169 169 292 292 292 F F F
185382024-02-07T19:02:46.352Z 27 540-559 169 169 169 292 292 292 F F F
185392024-02-07T19:02:46.352Z 28 560-579 168 168 168 290 290 290 F F F
185402024-02-07T19:02:46.352Z 29 580-599 168 168 168 288 288 288 F F F
185412024-02-07T19:02:46.352ZMax gen: 169, Max flush: 292
185422024-02-07T19:02:46.352Z
185432024-02-07T19:02:46.352ZBegin loop 85
185442024-02-07T19:02:46.352ZDownstairs to restart: 2
185452024-02-07T19:02:46.368Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.56873516Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815}
185462024-02-07T19:02:46.374Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569042639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815}
185472024-02-07T19:02:46.374Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569090283Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815}
185482024-02-07T19:02:46.374Z{"msg":"Crucible 10f74331-fc79-4f21-b5b6-6b4b6e5957fb has session id: c57f21b4-31d2-4233-9d93-9e84f4f43454","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569120918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185492024-02-07T19:02:46.374Z{"msg":"Upstairs opts: Upstairs UUID: 10f74331-fc79-4f21-b5b6-6b4b6e5957fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.56918567Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185502024-02-07T19:02:46.374Z{"msg":"Crucible stats registered with UUID: 10f74331-fc79-4f21-b5b6-6b4b6e5957fb","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569240152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185512024-02-07T19:02:46.374ZCrucible runtime is spawned
185522024-02-07T19:02:46.374Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569389322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815}
185532024-02-07T19:02:46.374Z{"msg":"10f74331-fc79-4f21-b5b6-6b4b6e5957fb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569427367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185542024-02-07T19:02:46.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569456563Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185552024-02-07T19:02:46.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569492308Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185562024-02-07T19:02:46.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569524084Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185572024-02-07T19:02:46.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569561179Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185582024-02-07T19:02:46.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569592265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185592024-02-07T19:02:46.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569618421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185602024-02-07T19:02:46.374Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569647577Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185612024-02-07T19:02:46.375Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569686102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185622024-02-07T19:02:46.375Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569805226Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185632024-02-07T19:02:46.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569856349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185642024-02-07T19:02:46.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.569888384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185652024-02-07T19:02:46.375Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:45.569942597Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185662024-02-07T19:02:46.375Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:45.569988871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185672024-02-07T19:02:46.375Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:45.570052552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185682024-02-07T19:02:46.375Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.570090767Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185692024-02-07T19:02:46.375Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.570121633Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185702024-02-07T19:02:46.375Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.570148819Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185712024-02-07T19:02:46.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.57058665Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185722024-02-07T19:02:46.375Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.570622425Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185732024-02-07T19:02:46.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.570637883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185742024-02-07T19:02:46.375Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.570686037Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185752024-02-07T19:02:46.375Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571194228Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185762024-02-07T19:02:46.375Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571340888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185772024-02-07T19:02:46.375Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571778559Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185782024-02-07T19:02:46.375Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571815294Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185792024-02-07T19:02:46.375Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571830572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185802024-02-07T19:02:46.375Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571857668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185812024-02-07T19:02:46.375Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571880365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185822024-02-07T19:02:46.375Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:45.571894823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185832024-02-07T19:02:47.371Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.571625339Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185842024-02-07T19:02:47.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.571712977Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185852024-02-07T19:02:47.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.572623534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185862024-02-07T19:02:47.376Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.572670508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185872024-02-07T19:02:47.376Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.573202896Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185882024-02-07T19:02:47.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.573825061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185892024-02-07T19:02:47.376Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.573867926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185902024-02-07T19:02:47.376Z{"msg":"[0]R flush_numbers[0..12]: [287, 286, 285, 292, 274, 287, 281, 276, 288, 288, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.573886853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185912024-02-07T19:02:47.376Z{"msg":"[0]R generation[0..12]: [168, 167, 166, 169, 160, 168, 164, 161, 168, 168, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.57390531Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185922024-02-07T19:02:47.376Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574007097Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185932024-02-07T19:02:47.376Z{"msg":"[1]R flush_numbers[0..12]: [287, 286, 285, 292, 274, 287, 281, 276, 288, 288, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574044592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185942024-02-07T19:02:47.376Z{"msg":"[1]R generation[0..12]: [168, 167, 166, 169, 160, 168, 164, 161, 168, 168, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574063289Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185952024-02-07T19:02:47.376Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574089826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185962024-02-07T19:02:47.376Z{"msg":"[2]R flush_numbers[0..12]: [287, 286, 285, 292, 274, 287, 281, 276, 288, 288, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574108133Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185972024-02-07T19:02:47.376Z{Wait for a query_work_queue command to finish before sending IO
185982024-02-07T19:02:47.376Z"msg":"[2]R generation[0..12]: [168, 167, 166, 169, 160, 168, 164, 161, 168, 168, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574138069Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
185992024-02-07T19:02:47.376Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574157696Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186002024-02-07T19:02:47.376Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574182993Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186012024-02-07T19:02:47.376Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.57420093Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186022024-02-07T19:02:47.377Z{"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574218158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186032024-02-07T19:02:47.377Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574234996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186042024-02-07T19:02:47.377Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574253563Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186052024-02-07T19:02:47.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574286889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186062024-02-07T19:02:47.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574312835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186072024-02-07T19:02:47.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.57435278Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186082024-02-07T19:02:47.377Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574371058Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186092024-02-07T19:02:47.377Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574387915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186102024-02-07T19:02:47.377Z{"msg":"10f74331-fc79-4f21-b5b6-6b4b6e5957fb is now active with session: c57f21b4-31d2-4233-9d93-9e84f4f43454","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574418311Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186112024-02-07T19:02:47.377Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574438449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815,"session_id":"c57f21b4-31d2-4233-9d93-9e84f4f43454"}
186122024-02-07T19:02:47.377Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:46.574454826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815}
186132024-02-07T19:02:47.377ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186142024-02-07T19:02:47.380ZLoading write count information from file "/tmp/test_repair_verify.data"
186152024-02-07T19:02:47.380ZRun Repair workload
186162024-02-07T19:02:47.380Z01/30 Write block 163 len 32768 data: 15 18 17 15 14 17 17 16
186172024-02-07T19:02:47.380Z02/30 Write block 509 len 24576 data: 14 16 14 18 16 19
186182024-02-07T19:02:47.381Z03/30 Read block 86 len 40960
186192024-02-07T19:02:47.381Z04/30 Read block 583 len 8192
186202024-02-07T19:02:47.381Z05/30 Write block 374 len 16384 data: 13 13 15 14
186212024-02-07T19:02:47.384Z06/30 Write block 493 len 28672 data: 12 16 10 14 11 13 12
186222024-02-07T19:02:47.384Z07/30 Read block 435 len 24576
186232024-02-07T19:02:47.384Z08/30 Read block 567 len 12288
186242024-02-07T19:02:47.384Z09/30 Flush
186252024-02-07T19:02:47.390Z10/30 Write block 29 len 40960 data: 19 18 19 19 18 20 18 18 13 12
186262024-02-07T19:02:47.390Z11/30 Write block 526 len 20480 data: 10 12 10 11 10
186272024-02-07T19:02:47.393Z12/30 Write block 529 len 28672 data: 12 11 10 11 11 12 15
186282024-02-07T19:02:47.393Z13/30 Write block 11 len 4096 data: 14
186292024-02-07T19:02:47.393Z14/30 Read block 300 len 12288
186302024-02-07T19:02:47.393Z15/30 Write block 417 len 24576 data: 10 12 9 10 10 9
186312024-02-07T19:02:47.393Z16/30 Write block 492 len 8192 data: 13 13
186322024-02-07T19:02:47.393Z17/30 Write block 52 len 28672 data: 10 11 12 14 14 12 10
186332024-02-07T19:02:47.393Z18/30 Write block 360 len 16384 data: 12 14 16 15
186342024-02-07T19:02:47.393Z19/30 Read block 506 len 32768
186352024-02-07T19:02:47.393Z20/30 Flush
186362024-02-07T19:02:47.396Z21/30 Write block 536 len 28672 data: 15 16 14 14 15 16 15
186372024-02-07T19:02:47.396Z22/30 Read block 344 len 32768
186382024-02-07T19:02:47.399Z23/30 Write block 432 len 8192 data: 13 11
186392024-02-07T19:02:47.399Z24/30 Flush
186402024-02-07T19:02:47.402Z25/30 Write block 411 len 16384 data: 15 11 13 9
186412024-02-07T19:02:47.402Z26/30 Flush
186422024-02-07T19:02:47.405Z27/30 Write block 556 len 28672 data: 14 11 12 14 15 12 14
186432024-02-07T19:02:47.405Z28/30 Write block 438 len 20480 data: 9 12 13 13 14
186442024-02-07T19:02:47.406Z29/30 Write block 515 len 28672 data: 19 17 14 16 14 14 12
186452024-02-07T19:02:47.406Z30/30 Read block 114 len 8192
186462024-02-07T19:02:47.409Z----------------------------------------------------------------
186472024-02-07T19:02:47.409Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
186482024-02-07T19:02:47.409ZStates: Active Active Active
186492024-02-07T19:02:47.409ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
186502024-02-07T19:02:47.409Z 1 Acked 1000 Write 8 Done Done Done false
186512024-02-07T19:02:47.409Z 2 Acked 1001 Write 6 Done Done Sent false
186522024-02-07T19:02:47.409Z 3 Acked 1002 Read 10 Done Done Sent false
186532024-02-07T19:02:47.409Z 4 Acked 1003 Read 2 Done Done Sent false
186542024-02-07T19:02:47.409Z 5 Acked 1004 Write 4 Done Done Sent false
186552024-02-07T19:02:47.409Z 6 Acked 1005 Write 7 Done Done Sent false
186562024-02-07T19:02:47.409Z 7 Acked 1006 Read 6 Done Done Sent false
186572024-02-07T19:02:47.409Z 8 Acked 1007 Read 3 Done Done Sent false
186582024-02-07T19:02:47.410Z 9 Acked 1008 Flush 0 Done Done Sent false
186592024-02-07T19:02:47.410Z 10 Acked 1009 Write 10 Done Done Sent false
186602024-02-07T19:02:47.410Z 11 Acked 1010 Write 5 Done Done Sent false
186612024-02-07T19:02:47.410Z 12 Acked 1011 Write 7 Done Done Sent false
186622024-02-07T19:02:47.410Z 13 Acked 1012 Write 1 Done Done Sent false
186632024-02-07T19:02:47.410Z 14 Acked 1013 Read 3 Done Done Sent false
186642024-02-07T19:02:47.410Z 15 Acked 1014 Write 6 Done Done Sent false
186652024-02-07T19:02:47.410Z 16 Acked 1015 Write 2 Done Done Sent false
186662024-02-07T19:02:47.410Z 17 Acked 1016 Write 7 Done Done Sent false
186672024-02-07T19:02:47.410Z 18 Acked 1017 Write 4 Done Done Sent false
186682024-02-07T19:02:47.410Z 19 Acked 1018 Read 8 Done Done Sent false
186692024-02-07T19:02:47.410Z 20 Acked 1019 Flush 0 Done Done Sent false
186702024-02-07T19:02:47.410Z 21 Acked 1020 Write 7 Done Done Sent false
186712024-02-07T19:02:47.410Z 22 Acked 1021 Read 8 Done Done Sent false
186722024-02-07T19:02:47.410Z 23 Acked 1022 Write 2 Done Done Sent false
186732024-02-07T19:02:47.410Z 24 Acked 1023 Flush 0 Done Done Sent false
186742024-02-07T19:02:47.410Z 25 Acked 1024 Write 4 Done Done Sent false
186752024-02-07T19:02:47.410Z 26 Acked 1025 Flush 0 Done Done Sent false
186762024-02-07T19:02:47.410Z 27 Acked 1026 Write 7 Done Done Sent false
186772024-02-07T19:02:47.410Z 28 Acked 1027 Write 5 Done Done Sent false
186782024-02-07T19:02:47.410Z 29 Acked 1028 Write 7 Done Done Sent false
186792024-02-07T19:02:47.410Z 30 Acked 1029 Read 2 Sent Done Sent false
186802024-02-07T19:02:47.410Z STATES DS:0 DS:1 DS:2 TOTAL
186812024-02-07T19:02:47.410Z New 0 0 0 0
186822024-02-07T19:02:47.410Z Sent 1 0 29 30
186832024-02-07T19:02:47.411Z Done 29 30 1 60
186842024-02-07T19:02:47.411Z Skipped 0 0 0 0
186852024-02-07T19:02:47.411Z Error 0 0 0 0
186862024-02-07T19:02:47.411ZLast Flush: 1025 1025 0
186872024-02-07T19:02:47.411ZDownstairs last five completed:
186882024-02-07T19:02:47.411ZUpstairs last five completed: 30 29 28 27 26
186892024-02-07T19:02:47.411Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:46.607003694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1815}
186902024-02-07T19:02:47.411ZWrote out file "/tmp/test_repair_verify.data"
186912024-02-07T19:02:47.414Z
186922024-02-07T19:02:47.417ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186932024-02-07T19:02:47.436ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186942024-02-07T19:02:47.436Z 0 000-019 170 170 168 294 294 287 F F F <---
186952024-02-07T19:02:47.436Z 1 020-039 170 170 167 294 294 286 F F F <---
186962024-02-07T19:02:47.436Z 2 040-059 170 170 166 294 294 285 F F F <---
186972024-02-07T19:02:47.437Z 3 060-079 169 169 169 292 292 292 F F F
186982024-02-07T19:02:47.437Z 4 080-099 160 160 160 274 274 274 F F F
186992024-02-07T19:02:47.437Z 5 100-119 168 168 168 287 287 287 F F F
187002024-02-07T19:02:47.449Z 6 120-139 164 164 164 281 281 281 F F F
187012024-02-07T19:02:47.450Z 7 140-159 161 161 161 276 276 276 F F F
187022024-02-07T19:02:47.450Z 8 160-179 170 170 168 293 293 288 F F T <---
187032024-02-07T19:02:47.450Z 9 180-199 168 168 168 288 288 288 F F F
187042024-02-07T19:02:47.450Z 10 200-219 160 160 160 274 274 274 F F F
187052024-02-07T19:02:47.450Z 11 220-239 166 166 166 285 285 285 F F F
187062024-02-07T19:02:47.450Z 12 240-259 167 167 167 286 286 286 F F F
187072024-02-07T19:02:47.450Z 13 260-279 164 164 164 282 282 282 F F F
187082024-02-07T19:02:47.450Z 14 280-299 169 169 169 292 292 292 F F F
187092024-02-07T19:02:47.451Z 15 300-319 166 166 166 285 285 285 F F F
187102024-02-07T19:02:47.451Z 16 320-339 169 169 169 292 292 292 F F F
187112024-02-07T19:02:47.451Z 17 340-359 169 169 169 292 292 292 F F F
187122024-02-07T19:02:47.451Z 18 360-379 170 170 164 294 294 281 F F F <---
187132024-02-07T19:02:47.451Z 19 380-399 164 164 164 281 281 281 F F F
187142024-02-07T19:02:47.451Z 20 400-419 170 170 145 296 296 245 F F F <---
187152024-02-07T19:02:47.451Z 21 420-439 170 170 168 295 295 288 T T F <---
187162024-02-07T19:02:47.451Z 22 440-459 168 168 168 287 287 287 T T F <---
187172024-02-07T19:02:47.451Z 23 460-479 166 166 166 285 285 285 F F F
187182024-02-07T19:02:47.451Z 24 480-499 170 170 161 294 294 276 F F F <---
187192024-02-07T19:02:47.451Z 25 500-519 170 170 169 293 293 292 T T F <---
187202024-02-07T19:02:47.452Z 26 520-539 170 170 169 295 295 292 T T F <---
187212024-02-07T19:02:47.452Z 27 540-559 170 170 169 295 295 292 T T F <---
187222024-02-07T19:02:47.452Z 28 560-579 168 168 168 290 290 290 T T F <---
187232024-02-07T19:02:47.452Z 29 580-599 168 168 168 288 288 288 F F F
187242024-02-07T19:02:47.452ZMax gen: 170, Max flush: 296
187252024-02-07T19:02:47.452ZError: Difference in extent metadata found!
187262024-02-07T19:02:47.452ZA Difference in extent metadata is expected here
187272024-02-07T19:02:47.452ZOn loop 85
187282024-02-07T19:02:47.452Z
187292024-02-07T19:02:47.452ZVerifying data now
187302024-02-07T19:02:53.116ZNo change after verify
187312024-02-07T19:02:53.116ZLoop: 85 Downstairs dump after verify (and repair):
187322024-02-07T19:02:53.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187332024-02-07T19:02:53.128Z 0 000-019 170 170 170 294 294 294 F F F
187342024-02-07T19:02:53.128Z 1 020-039 170 170 170 294 294 294 F F F
187352024-02-07T19:02:53.128Z 2 040-059 170 170 170 294 294 294 F F F
187362024-02-07T19:02:53.128Z 3 060-079 169 169 169 292 292 292 F F F
187372024-02-07T19:02:53.128Z 4 080-099 160 160 160 274 274 274 F F F
187382024-02-07T19:02:53.128Z 5 100-119 168 168 168 287 287 287 F F F
187392024-02-07T19:02:53.128Z 6 120-139 164 164 164 281 281 281 F F F
187402024-02-07T19:02:53.128Z 7 140-159 161 161 161 276 276 276 F F F
187412024-02-07T19:02:53.129Z 8 160-179 170 170 170 293 293 293 F F F
187422024-02-07T19:02:53.129Z 9 180-199 168 168 168 288 288 288 F F F
187432024-02-07T19:02:53.129Z 10 200-219 160 160 160 274 274 274 F F F
187442024-02-07T19:02:53.129Z 11 220-239 166 166 166 285 285 285 F F F
187452024-02-07T19:02:53.129Z 12 240-259 167 167 167 286 286 286 F F F
187462024-02-07T19:02:53.129Z 13 260-279 164 164 164 282 282 282 F F F
187472024-02-07T19:02:53.129Z 14 280-299 169 169 169 292 292 292 F F F
187482024-02-07T19:02:53.129Z 15 300-319 166 166 166 285 285 285 F F F
187492024-02-07T19:02:53.129Z 16 320-339 169 169 169 292 292 292 F F F
187502024-02-07T19:02:53.129Z 17 340-359 169 169 169 292 292 292 F F F
187512024-02-07T19:02:53.129Z 18 360-379 170 170 170 294 294 294 F F F
187522024-02-07T19:02:53.129Z 19 380-399 164 164 164 281 281 281 F F F
187532024-02-07T19:02:53.129Z 20 400-419 170 170 170 296 296 296 F F F
187542024-02-07T19:02:53.130Z 21 420-439 171 171 171 297 297 297 F F F
187552024-02-07T19:02:53.130Z 22 440-459 171 171 171 297 297 297 F F F
187562024-02-07T19:02:53.130Z 23 460-479 166 166 166 285 285 285 F F F
187572024-02-07T19:02:53.130Z 24 480-499 170 170 170 294 294 294 F F F
187582024-02-07T19:02:53.130Z 25 500-519 171 171 171 297 297 297 F F F
187592024-02-07T19:02:53.130Z 26 520-539 171 171 171 297 297 297 F F F
187602024-02-07T19:02:53.130Z 27 540-559 171 171 171 297 297 297 F F F
187612024-02-07T19:02:53.130Z 28 560-579 171 171 171 297 297 297 F F F
187622024-02-07T19:02:53.130Z 29 580-599 168 168 168 288 288 288 F F F
187632024-02-07T19:02:53.130ZMax gen: 171, Max flush: 297
187642024-02-07T19:02:53.130Z
187652024-02-07T19:02:53.130ZBegin loop 86
187662024-02-07T19:02:53.131ZDownstairs to restart: 1
187672024-02-07T19:02:53.139Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335191324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823}
187682024-02-07T19:02:53.139Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335479285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823}
187692024-02-07T19:02:53.147Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.33551372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823}
187702024-02-07T19:02:53.147Z{"msg":"Crucible 41d664e3-11f5-4ce2-8474-d0087e0f099a has session id: b4a67094-7cd4-48d1-9382-d172be8c9810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335547515Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187712024-02-07T19:02:53.147Z{"msg":"Upstairs opts: Upstairs UUID: 41d664e3-11f5-4ce2-8474-d0087e0f099a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335582971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187722024-02-07T19:02:53.147Z{"msg":"Crucible stats registered with UUID: 41d664e3-11f5-4ce2-8474-d0087e0f099a","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335608737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187732024-02-07T19:02:53.147ZCrucible runtime is spawned
187742024-02-07T19:02:53.147Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335871381Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823}
187752024-02-07T19:02:53.148Z{"msg":"41d664e3-11f5-4ce2-8474-d0087e0f099a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335932943Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187762024-02-07T19:02:53.148Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335965529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187772024-02-07T19:02:53.148Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.335981657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187782024-02-07T19:02:53.148Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336006893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187792024-02-07T19:02:53.148Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.33603076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187802024-02-07T19:02:53.148Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336052677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187812024-02-07T19:02:53.148Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336077413Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187822024-02-07T19:02:53.148Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336109859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187832024-02-07T19:02:53.148Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.33617693Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187842024-02-07T19:02:53.148Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336240171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187852024-02-07T19:02:53.148Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:52.336274417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187862024-02-07T19:02:53.148Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:52.336306423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187872024-02-07T19:02:53.148Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336343228Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187882024-02-07T19:02:53.149Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:52.336371274Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187892024-02-07T19:02:53.149Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.33640241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187902024-02-07T19:02:53.149Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336419697Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187912024-02-07T19:02:53.149Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336442494Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187922024-02-07T19:02:53.149Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.336599913Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187932024-02-07T19:02:53.149Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.337101625Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187942024-02-07T19:02:53.149Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.33713727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187952024-02-07T19:02:53.149Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.337186943Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187962024-02-07T19:02:53.149Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.337227638Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187972024-02-07T19:02:53.149Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.337738249Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187982024-02-07T19:02:53.149Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.337884539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
187992024-02-07T19:02:53.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.338275096Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188002024-02-07T19:02:53.149Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.338312631Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188012024-02-07T19:02:53.150Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.338329359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188022024-02-07T19:02:53.150Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.338478109Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188032024-02-07T19:02:53.150Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.338508225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188042024-02-07T19:02:53.150Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:52.338523013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188052024-02-07T19:02:57.428Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.337107823Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188062024-02-07T19:02:57.511Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.337297498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188072024-02-07T19:02:57.511Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.338226192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188082024-02-07T19:02:57.511Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.338263927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188092024-02-07T19:02:57.511Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.338750251Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188102024-02-07T19:02:57.511Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339377326Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188112024-02-07T19:02:57.511Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339402263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188122024-02-07T19:02:57.511Z{"msg":"[0]R flush_numbers[0..12]: [294, 294, 294, 292, 274, 287, 281, 276, 293, 288, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.33942156Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188132024-02-07T19:02:57.511Z{"msg":"[0]R generation[0..12]: [170, 170, 170, 169, 160, 168, 164, 161, 170, 168, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339440198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188142024-02-07T19:02:57.511Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339475083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188152024-02-07T19:02:57.511Z{"msg":"[1]R flush_numbers[0..12]: [294, 294, 294, 292, 274, 287, 281, 276, 293, 288, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339494011Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188162024-02-07T19:02:57.511Z{"msg":"[1]R generation[0..12]: [170, 170, 170, 169, 160, 168, 164, 161, 170, 168, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339512168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188172024-02-07T19:02:57.511Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339529826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188182024-02-07T19:02:57.511Z{"msg":"[2]R flush_numbers[0..12]: [294, 294, 294, 292, 274, 287, 281, 276, 293, 288, 274, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339556362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188192024-02-07T19:02:57.511Z{"msg":"[2]R generation[0..12]: [170, 170, 170, 169, 160, 168, 164, 161, 170, 168, 160, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.33957592Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188202024-02-07T19:02:57.512Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339602466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188212024-02-07T19:02:57.512Z{"msg":"Max found gen is 172","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
188222024-02-07T19:02:57.512Z,"level":30,"time":"2024-02-07T19:02:53.339623293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188232024-02-07T19:02:57.512Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339687444Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188242024-02-07T19:02:57.512Z{"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339704332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188252024-02-07T19:02:57.512Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.33971961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188262024-02-07T19:02:57.512Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339739757Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188272024-02-07T19:02:57.512Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339759285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188282024-02-07T19:02:57.512Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339781062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188292024-02-07T19:02:57.512Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339812737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188302024-02-07T19:02:57.512Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339843083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188312024-02-07T19:02:57.512Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.3398651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188322024-02-07T19:02:57.512Z{"msg":"41d664e3-11f5-4ce2-8474-d0087e0f099a is now active with session: b4a67094-7cd4-48d1-9382-d172be8c9810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339894086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188332024-02-07T19:02:57.512Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339907485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823,"session_id":"b4a67094-7cd4-48d1-9382-d172be8c9810"}
188342024-02-07T19:02:57.512Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:53.339937151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823}
188352024-02-07T19:02:57.512ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188362024-02-07T19:02:57.512ZLoading write count information from file "/tmp/test_repair_verify.data"
188372024-02-07T19:02:57.512ZRun Repair workload
188382024-02-07T19:02:57.512Z01/30 Write block 452 len 40960 data: 11 13 13 12 10 14 15 14 13 12
188392024-02-07T19:02:57.512Z02/30 Read block 35 len 24576
188402024-02-07T19:02:57.512Z03/30 Write block 555 len 12288 data: 15 15 12
188412024-02-07T19:02:57.513Z04/30 Flush
188422024-02-07T19:02:57.513Z05/30 Write block 501 len 32768 data: 10 14 16 14 17 15 13 14
188432024-02-07T19:02:57.513Z06/30 Read block 547 len 32768
188442024-02-07T19:02:57.513Z07/30 Read block 296 len 20480
188452024-02-07T19:02:57.513Z08/30 Read block 589 len 12288
188462024-02-07T19:02:57.513Z09/30 Flush
188472024-02-07T19:02:57.513Z10/30 Write block 316 len 36864 data: 17 14 14 11 10 9 10 9 8
188482024-02-07T19:02:57.513Z11/30 Read block 488 len 24576
188492024-02-07T19:02:57.513Z12/30 Read block 254 len 12288
188502024-02-07T19:02:57.513Z13/30 Write block 418 len 28672 data: 13 10 11 11 10 10 11
188512024-02-07T19:02:57.513Z14/30 Read block 364 len 8192
188522024-02-07T19:02:57.513Z15/30 Read block 216 len 20480
188532024-02-07T19:02:57.513Z16/30 Read block 511 len 36864
188542024-02-07T19:02:57.513Z17/30 Read block 570 len 36864
188552024-02-07T19:02:57.513Z18/30 Write block 389 len 36864 data: 15 9 8 8 11 10 9 11 12
188562024-02-07T19:02:57.513Z19/30 Flush
188572024-02-07T19:02:57.513Z20/30 Write block 166 len 24576 data: 16 15 18 18 17 14
188582024-02-07T19:02:57.513Z21/30 Read block 162 len 12288
188592024-02-07T19:02:57.513Z22/30 Read block 371 len 4096
188602024-02-07T19:02:57.513Z23/30 Write block 346 len 24576 data: 18 18 16 15 15 14
188612024-02-07T19:02:57.513Z24/30 Flush
188622024-02-07T19:02:57.513Z25/30 Write block 305 len 28672 data: 10 9 8 11 12 12 11
188632024-02-07T19:02:57.514Z26/30 Flush
188642024-02-07T19:02:57.514Z27/30 Write block 196 len 28672 data: 12 13 15 14 12 12 13
188652024-02-07T19:02:57.514Z28/30 Write block 97 len 4096 data: 16
188662024-02-07T19:02:57.514Z29/30 Write block 363 len 4096 data: 16
188672024-02-07T19:02:57.514Z30/30 Read block 372 len 32768
188682024-02-07T19:02:57.514Z----------------------------------------------------------------
188692024-02-07T19:02:57.514Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
188702024-02-07T19:02:57.514ZStates: Active Active Active
188712024-02-07T19:02:57.514ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
188722024-02-07T19:02:57.514Z 1 Acked 1000 Write 10 Done Sent Done false
188732024-02-07T19:02:57.514Z 2 Acked 1001 Read 6 Done Sent Done false
188742024-02-07T19:02:57.514Z 3 Acked 1002 Write 3 Done Sent Done false
188752024-02-07T19:02:57.514Z 4 Acked 1003 Flush 0 Done Sent Done false
188762024-02-07T19:02:57.514Z 5 Acked 1004 Write 8 Done Sent Done false
188772024-02-07T19:02:57.514Z 6 Acked 1005 Read 8 Done Sent Done false
188782024-02-07T19:02:57.514Z 7 Acked 1006 Read 5 Done Sent Done false
188792024-02-07T19:02:57.514Z 8 Acked 1007 Read 3 Done Sent Done false
188802024-02-07T19:02:57.514Z 9 Acked 1008 Flush 0 Done Sent Done false
188812024-02-07T19:02:57.514Z 10 Acked 1009 Write 9 Done Sent Done false
188822024-02-07T19:02:57.514Z 11 Acked 1010 Read 6 Done Sent Done false
188832024-02-07T19:02:57.514Z 12 Acked 1011 Read 3 Done Sent Done false
188842024-02-07T19:02:57.514Z 13 Acked 1012 Write 7 Done Sent Done false
188852024-02-07T19:02:57.514Z 14 Acked 1013 Read 2 Done Sent Done false
188862024-02-07T19:02:57.515Z 15 Acked 1014 Read 5 Done Sent Done false
188872024-02-07T19:02:57.515Z 16 Acked 1015 Read 9 Done Sent Done false
188882024-02-07T19:02:57.515Z 17 Acked 1016 Read 9 Done Sent Done false
188892024-02-07T19:02:57.515Z 18 Acked 1017 Write 9 Done Sent Done false
188902024-02-07T19:02:57.515Z 19 Acked 1018 Flush 0 Done Sent Done false
188912024-02-07T19:02:57.515Z 20 Acked 1019 Write 6 Done Sent Done false
188922024-02-07T19:02:57.515Z 21 Acked 1020 Read 3 Done Sent Done false
188932024-02-07T19:02:57.515Z 22 Acked 1021 Read 1 Done Sent Done false
188942024-02-07T19:02:57.515Z 23 Acked 1022 Write 6 Done Sent Done false
188952024-02-07T19:02:57.515Z 24 Acked 1023 Flush 0 Done Sent Done false
188962024-02-07T19:02:57.515Z 25 Acked 1024 Write 7 Done Sent Done false
188972024-02-07T19:02:57.515Z 26 Acked 1025 Flush 0 Done Sent Done false
188982024-02-07T19:02:57.515Z 27 Acked 1026 Write 7 Done Sent Done false
188992024-02-07T19:02:57.515Z 28 Acked 1027 Write 1 Done Sent Done false
189002024-02-07T19:02:57.515Z 29 Acked 1028 Write 1 Done Sent Done false
189012024-02-07T19:02:57.515Z 30 Acked 1029 Read 8 Sent Sent Done false
189022024-02-07T19:02:57.515Z STATES DS:0 DS:1 DS:2 TOTAL
189032024-02-07T19:02:57.515Z New 0 0 0 0
189042024-02-07T19:02:57.515Z Sent 1 30 0 31
189052024-02-07T19:02:57.515Z Done 29 0 30 59
189062024-02-07T19:02:57.550Z Skipped 0 0 0 0
189072024-02-07T19:02:57.551Z Error 0 0 0 0
189082024-02-07T19:02:57.551ZLast Flush: 1025 0 1025
189092024-02-07T19:02:57.551ZDownstairs last five completed:
189102024-02-07T19:02:57.551ZUpstairs last five completed: 30 29 28 27 26
189112024-02-07T19:02:57.551Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:53.376537321Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1823}
189122024-02-07T19:02:57.551ZWrote out file "/tmp/test_repair_verify.data"
189132024-02-07T19:02:57.551Z
189142024-02-07T19:02:57.551ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189152024-02-07T19:02:57.551ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189162024-02-07T19:02:57.551Z 0 000-019 170 170 170 294 294 294 F F F
189172024-02-07T19:02:57.551Z 1 020-039 170 170 170 294 294 294 F F F
189182024-02-07T19:02:57.551Z 2 040-059 170 170 170 294 294 294 F F F
189192024-02-07T19:02:57.551Z 3 060-079 169 169 169 292 292 292 F F F
189202024-02-07T19:02:57.551Z 4 080-099 160 160 160 274 274 274 T F T <---
189212024-02-07T19:02:57.551Z 5 100-119 168 168 168 287 287 287 F F F
189222024-02-07T19:02:57.551Z 6 120-139 164 164 164 281 281 281 F F F
189232024-02-07T19:02:57.551Z 7 140-159 161 161 161 276 276 276 F F F
189242024-02-07T19:02:57.551Z 8 160-179 172 170 172 301 293 301 F F F <---
189252024-02-07T19:02:57.551Z 9 180-199 168 168 168 288 288 288 T F T <---
189262024-02-07T19:02:57.551Z 10 200-219 160 160 160 274 274 274 T F T <---
189272024-02-07T19:02:57.551Z 11 220-239 166 166 166 285 285 285 F F F
189282024-02-07T19:02:57.552Z 12 240-259 167 167 167 286 286 286 F F F
189292024-02-07T19:02:57.552Z 13 260-279 164 164 164 282 282 282 F F F
189302024-02-07T19:02:57.552Z 14 280-299 169 169 169 292 292 292 F F F
189312024-02-07T19:02:57.552Z 15 300-319 172 166 172 302 285 302 F F F <---
189322024-02-07T19:02:57.552Z 16 320-339 172 169 172 300 292 300 F F F <---
189332024-02-07T19:02:57.552Z 17 340-359 172 169 172 301 292 301 F F F <---
189342024-02-07T19:02:57.552Z 18 360-379 170 170 170 294 294 294 T F T <---
189352024-02-07T19:02:57.552Z 19 380-399 172 164 172 300 281 300 F F F <---
189362024-02-07T19:02:57.552Z 20 400-419 172 170 172 300 296 300 F F F <---
189372024-02-07T19:02:57.552Z 21 420-439 172 171 172 300 297 300 F F F <---
189382024-02-07T19:02:57.552Z 22 440-459 172 171 172 298 297 298 F F F <---
189392024-02-07T19:02:57.552Z 23 460-479 172 166 172 298 285 298 F F F <---
189402024-02-07T19:02:57.552Z 24 480-499 170 170 170 294 294 294 F F F
189412024-02-07T19:02:57.552Z 25 500-519 172 171 172 299 297 299 F F F <---
189422024-02-07T19:02:57.552Z 26 520-539 171 171 171 297 297 297 F F F
189432024-02-07T19:02:57.552Z 27 540-559 172 171 172 298 297 298 F F F <---
189442024-02-07T19:02:57.552Z 28 560-579 171 171 171 297 297 297 F F F
189452024-02-07T19:02:57.552Z 29 580-599 168 168 168 288 288 288 F F F
189462024-02-07T19:02:57.552ZMax gen: 172, Max flush: 302
189472024-02-07T19:02:57.552ZError: Difference in extent metadata found!
189482024-02-07T19:02:57.552ZA Difference in extent metadata is expected here
189492024-02-07T19:02:57.553ZOn loop 86
189502024-02-07T19:02:57.553Z
189512024-02-07T19:02:57.553ZVerifying data now
189522024-02-07T19:02:58.920ZNo change after verify
189532024-02-07T19:02:58.920ZLoop: 86 Downstairs dump after verify (and repair):
189542024-02-07T19:02:58.935ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189552024-02-07T19:02:58.935Z 0 000-019 170 170 170 294 294 294 F F F
189562024-02-07T19:02:58.935Z 1 020-039 170 170 170 294 294 294 F F F
189572024-02-07T19:02:58.935Z 2 040-059 170 170 170 294 294 294 F F F
189582024-02-07T19:02:58.935Z 3 060-079 169 169 169 292 292 292 F F F
189592024-02-07T19:02:58.935Z 4 080-099 173 173 173 303 303 303 F F F
189602024-02-07T19:02:58.935Z 5 100-119 168 168 168 287 287 287 F F F
189612024-02-07T19:02:58.935Z 6 120-139 164 164 164 281 281 281 F F F
189622024-02-07T19:02:58.936Z 7 140-159 161 161 161 276 276 276 F F F
189632024-02-07T19:02:58.936Z 8 160-179 172 172 172 301 301 301 F F F
189642024-02-07T19:02:58.936Z 9 180-199 173 173 173 303 303 303 F F F
189652024-02-07T19:02:58.936Z 10 200-219 173 173 173 303 303 303 F F F
189662024-02-07T19:02:58.936Z 11 220-239 166 166 166 285 285 285 F F F
189672024-02-07T19:02:58.978Z 12 240-259 167 167 167 286 286 286 F F F
189682024-02-07T19:02:58.978Z 13 260-279 164 164 164 282 282 282 F F F
189692024-02-07T19:02:58.978Z 14 280-299 169 169 169 292 292 292 F F F
189702024-02-07T19:02:58.978Z 15 300-319 172 172 172 302 302 302 F F F
189712024-02-07T19:02:58.978Z 16 320-339 172 172 172 300 300 300 F F F
189722024-02-07T19:02:58.978Z 17 340-359 172 172 172 301 301 301 F F F
189732024-02-07T19:02:58.978Z 18 360-379 173 173 173 303 303 303 F F F
189742024-02-07T19:02:58.978Z 19 380-399 172 172 172 300 300 300 F F F
189752024-02-07T19:02:58.978Z 20 400-419 172 172 172 300 300 300 F F F
189762024-02-07T19:02:58.978Z 21 420-439 172 172 172 300 300 300 F F F
189772024-02-07T19:02:58.978Z 22 440-459 172 172 172 298 298 298 F F F
189782024-02-07T19:02:58.978Z 23 460-479 172 172 172 298 298 298 F F F
189792024-02-07T19:02:58.978Z 24 480-499 170 170 170 294 294 294 F F F
189802024-02-07T19:02:58.978Z 25 500-519 172 172 172 299 299 299 F F F
189812024-02-07T19:02:58.978Z 26 520-539 171 171 171 297 297 297 F F F
189822024-02-07T19:02:58.978Z 27 540-559 172 172 172 298 298 298 F F F
189832024-02-07T19:02:58.978Z 28 560-579 171 171 171 297 297 297 F F F
189842024-02-07T19:02:58.978Z 29 580-599 168 168 168 288 288 288 F F F
189852024-02-07T19:02:58.978ZMax gen: 173, Max flush: 303
189862024-02-07T19:02:58.978Z
189872024-02-07T19:02:58.978ZBegin loop 87
189882024-02-07T19:02:58.979ZDownstairs to restart: 2
189892024-02-07T19:02:58.994Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15493428Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831}
189902024-02-07T19:02:58.994Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155202964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831}
189912024-02-07T19:02:58.994Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
189922024-02-07T19:02:58.994Z,"time":"2024-02-07T19:02:58.155241688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831}
189932024-02-07T19:02:58.994Z{"msg":"Crucible ff43089e-7cd6-4a87-8263-a677f2179f1b has session id: 2c575ce5-fc2a-40e4-a468-4c8745b1edb4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155276244Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
189942024-02-07T19:02:58.994Z{"msg":"Upstairs opts: Upstairs UUID: ff43089e-7cd6-4a87-8263-a677f2179f1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155320548Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
189952024-02-07T19:02:58.994Z{"msg":"Crucible stats registered with UUID: ff43089e-7cd6-4a87-8263-a677f2179f1b","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155356283Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
189962024-02-07T19:02:58.994Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155388688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831}
189972024-02-07T19:02:58.994Z{"msg":"ff43089e-7cd6-4a87-8263-a677f2179f1b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155407906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
189982024-02-07T19:02:58.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155430653Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
189992024-02-07T19:02:58.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155463078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190002024-02-07T19:02:58.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155486515Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190012024-02-07T19:02:58.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155529659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190022024-02-07T19:02:58.995Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155557356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190032024-02-07T19:02:58.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155583482Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190042024-02-07T19:02:58.995Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155676529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190052024-02-07T19:02:58.995Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155723853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190062024-02-07T19:02:58.995Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155760328Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190072024-02-07T19:02:58.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155805932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190082024-02-07T19:02:58.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155838738Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190092024-02-07T19:02:58.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.155887671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190102024-02-07T19:02:58.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.156478451Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190112024-02-07T19:02:58.995Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.156518906Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190122024-02-07T19:02:58.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.156547402Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190132024-02-07T19:02:58.995Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.156575258Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190142024-02-07T19:02:58.995Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15722699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190152024-02-07T19:02:58.995Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.157269034Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190162024-02-07T19:02:58.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.157572173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190172024-02-07T19:02:58.995Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15766934Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190182024-02-07T19:02:58.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15788705Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190192024-02-07T19:02:58.995Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.157915057Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190202024-02-07T19:02:58.995Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.157929935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190212024-02-07T19:02:58.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.157952042Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190222024-02-07T19:02:58.996Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15796671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190232024-02-07T19:02:58.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.157986927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190242024-02-07T19:02:58.996Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158174512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190252024-02-07T19:02:58.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158624261Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190262024-02-07T19:02:58.996Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158687052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190272024-02-07T19:02:58.996Z{"msg":"[0]R flush_numbers[0..12]: [294, 294, 294, 292, 303, 287, 281, 276, 301, 303, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158726937Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190282024-02-07T19:02:58.996Z{"msg":"[0]R generation[0..12]: [170, 170, 170, 169, 173, 168, 164, 161, 172, 173, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158744254Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190292024-02-07T19:02:58.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158766332Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190302024-02-07T19:02:58.996Z{"msg":"[1]R flush_numbers[0..12]: [294, 294, 294, 292, 303, 287, 281, 276, 301, 303, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158781409Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190312024-02-07T19:02:58.996Z{"msg":"[1]R generation[0..12]: [170, 170, 170, 169, 173, 168, 164, 161, 172, 173, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158811996Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190322024-02-07T19:02:58.996ZWait for a query_work_queue command to finish before sending IO
190332024-02-07T19:02:58.996Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15885208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190342024-02-07T19:02:58.996Z{"msg":"[2]R flush_numbers[0..12]: [294, 294, 294, 292, 303, 287, 281, 276, 301, 303, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158889485Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190352024-02-07T19:02:58.996Z{"msg":"[2]R generation[0..12]: [170, 170, 170, 169, 173, 168, 164, 161, 172, 173, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158913722Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190362024-02-07T19:02:58.996Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.15892883Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190372024-02-07T19:02:58.996Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.158961955Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190382024-02-07T19:02:58.996Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159016758Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190392024-02-07T19:02:58.996Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159057912Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190402024-02-07T19:02:58.996Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159088438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190412024-02-07T19:02:58.996Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159103486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190422024-02-07T19:02:58.996Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159127573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190432024-02-07T19:02:58.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159163438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190442024-02-07T19:02:58.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159193724Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190452024-02-07T19:02:58.997Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159230059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190462024-02-07T19:02:58.997Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159256035Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190472024-02-07T19:02:58.997Z{"msg":"ff43089e-7cd6-4a87-8263-a677f2179f1b is now active with session: 2c575ce5-fc2a-40e4-a468-4c8745b1edb4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159270194Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190482024-02-07T19:02:58.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
190492024-02-07T19:02:58.997Z"time":"2024-02-07T19:02:58.15929469Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831,"session_id":"2c575ce5-fc2a-40e4-a468-4c8745b1edb4"}
190502024-02-07T19:02:58.997Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:02:58.159333055Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831}
190512024-02-07T19:02:58.997ZLoading write count information from file "/tmp/test_repair_verify.data"
190522024-02-07T19:02:58.997ZRun Repair workload
190532024-02-07T19:02:58.997Z01/30 Write block 532 len 28672 data: 12 12 13 16 16 17 15
190542024-02-07T19:02:58.997Z02/30 Read block 98 len 4096
190552024-02-07T19:02:58.997Z03/30 Read block 538 len 36864
190562024-02-07T19:02:58.997Z04/30 Flush
190572024-02-07T19:02:58.997Z05/30 Write block 32 len 32768 data: 20 19 21 19 19 14 13 10
190582024-02-07T19:02:58.997Z06/30 Read block 184 len 4096
190592024-02-07T19:02:58.997Z07/30 Read block 376 len 28672
190602024-02-07T19:02:58.997Z08/30 Write block 10 len 32768 data: 13 15 13 12 15 13 15 18
190612024-02-07T19:02:58.997Z09/30 Read block 307 len 24576
190622024-02-07T19:02:58.998Z10/30 Read block 363 len 16384
190632024-02-07T19:02:58.998Z11/30 Write block 378 len 8192 data: 14 11
190642024-02-07T19:02:58.998Z12/30 Write block 98 len 16384 data: 14 13 11 11
190652024-02-07T19:02:58.998Z13/30 Write block 167 len 8192 data: 16 19
190662024-02-07T19:02:58.998Z14/30 Read block 465 len 40960
190672024-02-07T19:02:58.998Z15/30 Write block 486 len 40960 data: 9 10 12 11 11 11 14 14 17 11
190682024-02-07T19:02:58.998Z16/30 Flush
190692024-02-07T19:02:58.998Z17/30 Write block 122 len 32768 data: 18 19 21 21 19 17 18 18
190702024-02-07T19:02:58.998Z18/30 Read block 362 len 36864
190712024-02-07T19:02:58.998Z19/30 Flush
190722024-02-07T19:02:58.998Z20/30 Write block 127 len 40960 data: 18 19 19 14 12 11 11 11 9 6
190732024-02-07T19:02:58.998Z21/30 Read block 584 len 24576
190742024-02-07T19:02:58.998Z22/30 Read block 91 len 40960
190752024-02-07T19:02:58.998Z23/30 Read block 37 len 4096
190762024-02-07T19:02:58.998Z24/30 Read block 342 len 40960
190772024-02-07T19:02:58.998Z25/30 Read block 463 len 32768
190782024-02-07T19:02:59.020Z26/30 Write block 249 len 24576 data: 13 12 12 13 12 15
190792024-02-07T19:02:59.020Z27/30 Write block 185 len 4096 data: 9
190802024-02-07T19:02:59.020Z28/30 Read block 367 len 8192
190812024-02-07T19:02:59.020Z29/30 Read block 415 len 40960
190822024-02-07T19:02:59.020Z30/30 Read block 442 len 36864
190832024-02-07T19:02:59.020Z----------------------------------------------------------------
190842024-02-07T19:02:59.020Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
190852024-02-07T19:02:59.020ZStates: Active Active Active
190862024-02-07T19:02:59.020ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190872024-02-07T19:02:59.020Z 1 Acked 1000 Write 7 Done Done Sent false
190882024-02-07T19:02:59.020Z 2 Acked 1001 Read 1 Done Done Sent false
190892024-02-07T19:02:59.020Z 3 Acked 1002 Read 9 Done Done Sent false
190902024-02-07T19:02:59.020Z 4 Acked 1003 Flush 0 Done Done Sent false
190912024-02-07T19:02:59.020Z 5 Acked 1004 Write 8 Done Done Sent false
190922024-02-07T19:02:59.020Z 6 Acked 1005 Read 1 Done Done Sent false
190932024-02-07T19:02:59.020Z 7 Acked 1006 Read 7 Done Done Sent false
190942024-02-07T19:02:59.020Z 8 Acked 1007 Write 8 Done Done Sent false
190952024-02-07T19:02:59.020Z 9 Acked 1008 Read 6 Done Done Sent false
190962024-02-07T19:02:59.020Z 10 Acked 1009 Read 4 Done Done Sent false
190972024-02-07T19:02:59.020Z 11 Acked 1010 Write 2 Done Done Sent false
190982024-02-07T19:02:59.021Z 12 Acked 1011 Write 4 Done Done Sent false
190992024-02-07T19:02:59.021Z 13 Acked 1012 Write 2 Done Done Sent false
191002024-02-07T19:02:59.021Z 14 Acked 1013 Read 10 Done Done Sent false
191012024-02-07T19:02:59.021Z 15 Acked 1014 Write 10 Done Done Sent false
191022024-02-07T19:02:59.021Z 16 Acked 1015 Flush 0 Done Done Sent false
191032024-02-07T19:02:59.021Z 17 Acked 1016 Write 8 Done Done Sent false
191042024-02-07T19:02:59.021Z 18 Acked 1017 Read 9 Done Done Sent false
191052024-02-07T19:02:59.021Z 19 Acked 1018 Flush 0 Done Done Sent false
191062024-02-07T19:02:59.021Z 20 Acked 1019 Write 10 Done Done Sent false
191072024-02-07T19:02:59.021Z 21 Acked 1020 Read 6 Done Done Sent false
191082024-02-07T19:02:59.021Z 22 Acked 1021 Read 10 Done Done Sent false
191092024-02-07T19:02:59.021Z 23 Acked 1022 Read 1 Done Done Sent false
191102024-02-07T19:02:59.021Z 24 Acked 1023 Read 10 Done Done Sent false
191112024-02-07T19:02:59.021Z 25 Acked 1024 Read 8 Done Done Sent false
191122024-02-07T19:02:59.021Z 26 Acked 1025 Write 6 Done Done Sent false
191132024-02-07T19:02:59.021Z 27 Acked 1026 Write 1 Done Done Sent false
191142024-02-07T19:02:59.021Z 28 Acked 1027 Read 2 Done Done Sent false
191152024-02-07T19:02:59.021Z 29 Acked 1028 Read 10 Done Done Sent false
191162024-02-07T19:02:59.021Z 30 Acked 1029 Read 9 Done Done Sent false
191172024-02-07T19:02:59.021Z STATES DS:0 DS:1 DS:2 TOTAL
191182024-02-07T19:02:59.022Z New 0 0 0 0
191192024-02-07T19:02:59.022Z Sent 0 0 30 30
191202024-02-07T19:02:59.022Z Done 30 30 0 60
191212024-02-07T19:02:59.022Z Skipped 0 0 0 0
191222024-02-07T19:02:59.022Z Error 0 0 0 0
191232024-02-07T19:02:59.022ZLast Flush: 1018 1018 0
191242024-02-07T19:02:59.022ZDownstairs last five completed:
191252024-02-07T19:02:59.022ZUpstairs last five completed: 30 29 28 27 26
191262024-02-07T19:02:59.022Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:02:58.19360346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1831}
191272024-02-07T19:02:59.022ZWrote out file "/tmp/test_repair_verify.data"
191282024-02-07T19:02:59.022Z
191292024-02-07T19:02:59.022ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
191302024-02-07T19:02:59.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191312024-02-07T19:02:59.027Z 0 000-019 174 174 170 305 305 294 F F F <---
191322024-02-07T19:02:59.027Z 1 020-039 174 174 170 305 305 294 F F F <---
191332024-02-07T19:02:59.027Z 2 040-059 170 170 170 294 294 294 F F F
191342024-02-07T19:02:59.027Z 3 060-079 169 169 169 292 292 292 F F F
191352024-02-07T19:02:59.027Z 4 080-099 174 174 173 305 305 303 F F F <---
191362024-02-07T19:02:59.027Z 5 100-119 174 174 168 305 305 287 F F F <---
191372024-02-07T19:02:59.027Z 6 120-139 174 174 164 306 306 281 T T F <---
191382024-02-07T19:02:59.027Z 7 140-159 161 161 161 276 276 276 F F F
191392024-02-07T19:02:59.027Z 8 160-179 174 174 172 305 305 301 F F F <---
191402024-02-07T19:02:59.028Z 9 180-199 173 173 173 303 303 303 T T F <---
191412024-02-07T19:02:59.028Z 10 200-219 173 173 173 303 303 303 F F F
191422024-02-07T19:02:59.028Z 11 220-239 166 166 166 285 285 285 F F F
191432024-02-07T19:02:59.028Z 12 240-259 167 167 167 286 286 286 T T F <---
191442024-02-07T19:02:59.028Z 13 260-279 164 164 164 282 282 282 F F F
191452024-02-07T19:02:59.028Z 14 280-299 169 169 169 292 292 292 F F F
191462024-02-07T19:02:59.028Z 15 300-319 172 172 172 302 302 302 F F F
191472024-02-07T19:02:59.028Z 16 320-339 172 172 172 300 300 300 F F F
191482024-02-07T19:02:59.028Z 17 340-359 172 172 172 301 301 301 F F F
191492024-02-07T19:02:59.028Z 18 360-379 174 174 173 305 305 303 F F F <---
191502024-02-07T19:02:59.028Z 19 380-399 172 172 172 300 300 300 F F F
191512024-02-07T19:02:59.028Z 20 400-419 172 172 172 300 300 300 F F F
191522024-02-07T19:02:59.028Z 21 420-439 172 172 172 300 300 300 F F F
191532024-02-07T19:02:59.028Z 22 440-459 172 172 172 298 298 298 F F F
191542024-02-07T19:02:59.028Z 23 460-479 172 172 172 298 298 298 F F F
191552024-02-07T19:02:59.028Z 24 480-499 174 174 170 305 305 294 F F F <---
191562024-02-07T19:02:59.028Z 25 500-519 172 172 172 299 299 299 F F F
191572024-02-07T19:02:59.028Z 26 520-539 174 174 171 304 304 297 F F F <---
191582024-02-07T19:02:59.028Z 27 540-559 172 172 172 298 298 298 F F F
191592024-02-07T19:02:59.028Z 28 560-579 171 171 171 297 297 297 F F F
191602024-02-07T19:02:59.028Z 29 580-599 168 168 168 288 288 288 F F F
191612024-02-07T19:02:59.028ZMax gen: 174, Max flush: 306
191622024-02-07T19:02:59.029ZError: Difference in extent metadata found!
191632024-02-07T19:02:59.029ZA Difference in extent metadata is expected here
191642024-02-07T19:02:59.029ZOn loop 87
191652024-02-07T19:02:59.029Z
191662024-02-07T19:02:59.029ZVerifying data now
191672024-02-07T19:03:03.583ZNo change after verify
191682024-02-07T19:03:03.583ZLoop: 87 Downstairs dump after verify (and repair):
191692024-02-07T19:03:03.598ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191702024-02-07T19:03:03.598Z 0 000-019 174 174 174 305 305 305 F F F
191712024-02-07T19:03:03.598Z 1 020-039 174 174 174 305 305 305 F F F
191722024-02-07T19:03:03.598Z 2 040-059 170 170 170 294 294 294 F F F
191732024-02-07T19:03:03.598Z 3 060-079 169 169 169 292 292 292 F F F
191742024-02-07T19:03:03.598Z 4 080-099 174 174 174 305 305 305 F F F
191752024-02-07T19:03:03.598Z 5 100-119 174 174 174 305 305 305 F F F
191762024-02-07T19:03:03.604Z 6 120-139 175 175 175 307 307 307 F F F
191772024-02-07T19:03:03.604Z 7 140-159 161 161 161 276 276 276 F F F
191782024-02-07T19:03:03.604Z 8 160-179 174 174 174 305 305 305 F F F
191792024-02-07T19:03:03.604Z 9 180-199 175 175 175 307 307 307 F F F
191802024-02-07T19:03:03.604Z 10 200-219 173 173 173 303 303 303 F F F
191812024-02-07T19:03:03.604Z 11 220-239 166 166 166 285 285 285 F F F
191822024-02-07T19:03:03.604Z 12 240-259 175 175 175 307 307 307 F F F
191832024-02-07T19:03:03.604Z 13 260-279 164 164 164 282 282 282 F F F
191842024-02-07T19:03:03.604Z 14 280-299 169 169 169 292 292 292 F F F
191852024-02-07T19:03:03.604Z 15 300-319 172 172 172 302 302 302 F F F
191862024-02-07T19:03:03.604Z 16 320-339 172 172 172 300 300 300 F F F
191872024-02-07T19:03:03.604Z 17 340-359 172 172 172 301 301 301 F F F
191882024-02-07T19:03:03.604Z 18 360-379 174 174 174 305 305 305 F F F
191892024-02-07T19:03:03.604Z 19 380-399 172 172 172 300 300 300 F F F
191902024-02-07T19:03:03.604Z 20 400-419 172 172 172 300 300 300 F F F
191912024-02-07T19:03:03.604Z 21 420-439 172 172 172 300 300 300 F F F
191922024-02-07T19:03:03.604Z 22 440-459 172 172 172 298 298 298 F F F
191932024-02-07T19:03:03.604Z 23 460-479 172 172 172 298 298 298 F F F
191942024-02-07T19:03:03.604Z 24 480-499 174 174 174 305 305 305 F F F
191952024-02-07T19:03:03.605Z 25 500-519 172 172 172 299 299 299 F F F
191962024-02-07T19:03:03.605Z 26 520-539 174 174 174 304 304 304 F F F
191972024-02-07T19:03:03.605Z 27 540-559 172 172 172 298 298 298 F F F
191982024-02-07T19:03:03.605Z 28 560-579 171 171 171 297 297 297 F F F
191992024-02-07T19:03:03.605Z 29 580-599 168 168 168 288 288 288 F F F
192002024-02-07T19:03:03.605ZMax gen: 175, Max flush: 307
192012024-02-07T19:03:03.605Z
192022024-02-07T19:03:03.605ZBegin loop 88
192032024-02-07T19:03:03.605ZDownstairs to restart: 1
192042024-02-07T19:03:03.620Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817296773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839}
192052024-02-07T19:03:03.620Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817550138Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839}
192062024-02-07T19:03:03.620Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817569976Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839}
192072024-02-07T19:03:03.628Z{"msg":"Crucible 7493894e-cfe0-4956-8554-910939dff222 has session id: dba24687-87a0-4b74-9d0b-a8ea09ce3bf1","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817604371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192082024-02-07T19:03:03.628Z{"msg":"Upstairs opts: Upstairs UUID: 7493894e-cfe0-4956-8554-910939dff222, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817644155Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192092024-02-07T19:03:03.629Z{"msg":"Crucible runtime is spawned
192102024-02-07T19:03:03.629ZCrucible stats registered with UUID: 7493894e-cfe0-4956-8554-910939dff222","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817677901Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192112024-02-07T19:03:03.629Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817764709Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839}
192122024-02-07T19:03:03.629Z{"msg":"7493894e-cfe0-4956-8554-910939dff222 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817854017Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192132024-02-07T19:03:03.629Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817882163Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192142024-02-07T19:03:03.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817899481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192152024-02-07T19:03:03.629Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817940706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192162024-02-07T19:03:03.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.817967012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192172024-02-07T19:03:03.629Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818009146Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192182024-02-07T19:03:03.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818038182Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192192024-02-07T19:03:03.629Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818063659Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192202024-02-07T19:03:03.629Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818092825Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192212024-02-07T19:03:03.629Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818120731Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192222024-02-07T19:03:03.629Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818195661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192232024-02-07T19:03:03.629Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818401243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192242024-02-07T19:03:03.629Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818460945Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192252024-02-07T19:03:03.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.818959548Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192262024-02-07T19:03:03.629Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.819011791Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192272024-02-07T19:03:03.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.819085251Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192282024-02-07T19:03:03.629Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.81916059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192292024-02-07T19:03:03.629Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.819545169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192302024-02-07T19:03:03.629Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.819696108Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192312024-02-07T19:03:03.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820091484Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192322024-02-07T19:03:03.630Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820135958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192332024-02-07T19:03:03.630Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820164385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192342024-02-07T19:03:03.630Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820224666Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192352024-02-07T19:03:03.630Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820253493Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192362024-02-07T19:03:03.630Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820364918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192372024-02-07T19:03:03.630Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820405362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192382024-02-07T19:03:03.630Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.820435858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192392024-02-07T19:03:03.630Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.82079448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192402024-02-07T19:03:03.630Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821257607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192412024-02-07T19:03:03.630Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821295532Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192422024-02-07T19:03:03.630Z{"msg":"[0]R flush_numbers[0..12]: [305, 305, 294, 292, 305, 305, 307, 276, 305, 307, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.82131205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192432024-02-07T19:03:03.630Z{"msg":"[0]R generation[0..12]: [174, 174, 170, 169, 174, 174, 175, 161, 174, 175, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821334816Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192442024-02-07T19:03:03.630Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821350214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192452024-02-07T19:03:03.630Z{"msg":"[1]R flush_numbers[0..12]: [305, 305, 294, 292, 305, 305, 307, 276, 305, 307, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821371981Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192462024-02-07T19:03:03.630Z{"msg":"[1]R generation[0..12]: [174, 174, 170, 169, 174, 174, 175, 161, 174, 175, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821387589Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192472024-02-07T19:03:03.630Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821409496Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192482024-02-07T19:03:03.630Z{"msg":"[2]R flush_numbers[0..12]: [305, 305, 294, 292, 305, 305, 307, 276, 305, 307, 303, 285]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
192492024-02-07T19:03:03.630Z,"time":"2024-02-07T19:03:02.821424574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192502024-02-07T19:03:03.630Z{"msg":"[2]R generation[0..12]: [174, 174, 170, 169, 174, 174, 175, 161, 174, 175, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821451281Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192512024-02-07T19:03:03.630Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821472508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192522024-02-07T19:03:03.630Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821487066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192532024-02-07T19:03:03.630Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821507263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192542024-02-07T19:03:03.630Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821520651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192552024-02-07T19:03:03.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821535439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192562024-02-07T19:03:03.630Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821549487Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192572024-02-07T19:03:03.631Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821569105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192582024-02-07T19:03:03.631Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821583663Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192592024-02-07T19:03:03.631Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.8216038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192602024-02-07T19:03:03.631Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821617848Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192612024-02-07T19:03:03.631Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821631316Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192622024-02-07T19:03:03.631Z{"msg":"7493894e-cfe0-4956-8554-910939dff222 is now active with session: dba24687-87a0-4b74-9d0b-a8ea09ce3bf1","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821651524Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192632024-02-07T19:03:03.631Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821664142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839,"session_id":"dba24687-87a0-4b74-9d0b-a8ea09ce3bf1"}
192642024-02-07T19:03:03.631Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:02.821683569Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839}
192652024-02-07T19:03:03.631ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
192662024-02-07T19:03:03.635ZLoading write count information from file "/tmp/test_repair_verify.data"
192672024-02-07T19:03:03.635ZRun Repair workload
192682024-02-07T19:03:03.635Z01/30 Write block 592 len 4096 data: 15
192692024-02-07T19:03:03.635Z02/30 Read block 480 len 16384
192702024-02-07T19:03:03.635Z03/30 Write block 125 len 40960 data: 22 20 19 20 20 15 13 12 12 12
192712024-02-07T19:03:03.635Z04/30 Read block 96 len 24576
192722024-02-07T19:03:03.635Z05/30 Write block 582 len 20480 data: 16 14 15 16 15
192732024-02-07T19:03:03.635Z06/30 Read block 516 len 16384
192742024-02-07T19:03:03.635Z07/30 Write block 565 len 28672 data: 18 18 19 17 16 17 15
192752024-02-07T19:03:03.635Z08/30 Write block 173 len 28672 data: 13 11 11 10 11 11 11
192762024-02-07T19:03:03.635Z09/30 Read block 267 len 36864
192772024-02-07T19:03:03.635Z10/30 Read block 38 len 16384
192782024-02-07T19:03:03.638Z11/30 Write block 517 len 24576 data: 15 17 15 15 13 8
192792024-02-07T19:03:03.639Z12/30 Flush
192802024-02-07T19:03:03.639Z13/30 Write block 334 len 12288 data: 17 16 16
192812024-02-07T19:03:03.642Z14/30 Write block 258 len 28672 data: 8 10 11 10 13 17 19
192822024-02-07T19:03:03.642Z15/30 Flush
192832024-02-07T19:03:03.642Z16/30 Write block 96 len 32768 data: 18 17 15 14 12 12 11 13
192842024-02-07T19:03:03.645Z17/30 Write block 434 len 16384 data: 8 9 10 13
192852024-02-07T19:03:03.645Z18/30 Flush
192862024-02-07T19:03:03.650Z19/30 Write block 585 len 24576 data: 17 16 12 15 13 15
192872024-02-07T19:03:03.650Z20/30 Write block 412 len 24576 data: 12 14 10 9 6 11
192882024-02-07T19:03:03.650Z21/30 Write block 54 len 24576 data: 13 15 15 13 11 10
192892024-02-07T19:03:03.650Z22/30 Write block 13 len 40960 data: 13 16 14 16 19 16 19 20 20 18
192902024-02-07T19:03:03.650Z23/30 Read block 129 len 12288
192912024-02-07T19:03:03.650Z24/30 Read block 461 len 28672
192922024-02-07T19:03:03.655Z25/30 Read block 212 len 16384
192932024-02-07T19:03:03.655Z26/30 Read block 545 len 40960
192942024-02-07T19:03:03.655Z27/30 Read block 66 len 16384
192952024-02-07T19:03:03.655Z28/30 Read block 15 len 4096
192962024-02-07T19:03:03.655Z29/30 Read block 290 len 20480
192972024-02-07T19:03:03.659Z30/30 Read block 293 len 20480
192982024-02-07T19:03:03.659Z----------------------------------------------------------------
192992024-02-07T19:03:03.659Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
193002024-02-07T19:03:03.659ZStates: Active Active Active
193012024-02-07T19:03:03.660ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193022024-02-07T19:03:03.660Z 1 Acked 1000 Write 1 Done Done Done false
193032024-02-07T19:03:03.660Z 2 Acked 1001 Read 4 Done Done Done false
193042024-02-07T19:03:03.660Z 3 Acked 1002 Write 10 Done Done Done false
193052024-02-07T19:03:03.660Z 4 Acked 1003 Read 6 Done Done Done false
193062024-02-07T19:03:03.660Z 5 Acked 1004 Write 5 Done Sent Done false
193072024-02-07T19:03:03.660Z 6 Acked 1005 Read 4 Done Sent Done false
193082024-02-07T19:03:03.660Z 7 Acked 1006 Write 7 Done Sent Done false
193092024-02-07T19:03:03.660Z 8 Acked 1007 Write 7 Done Sent Done false
193102024-02-07T19:03:03.660Z 9 Acked 1008 Read 9 Done Sent Done false
193112024-02-07T19:03:03.660Z 10 Acked 1009 Read 4 Done Sent Done false
193122024-02-07T19:03:03.660Z 11 Acked 1010 Write 6 Done Sent Done false
193132024-02-07T19:03:03.660Z 12 Acked 1011 Flush 0 Done Sent Done false
193142024-02-07T19:03:03.660Z 13 Acked 1012 Write 3 Done Sent Done false
193152024-02-07T19:03:03.660Z 14 Acked 1013 Write 7 Done Sent Done false
193162024-02-07T19:03:03.660Z 15 Acked 1014 Flush 0 Done Sent Done false
193172024-02-07T19:03:03.660Z 16 Acked 1015 Write 8 Done Sent Done false
193182024-02-07T19:03:03.660Z 17 Acked 1016 Write 4 Done Sent Done false
193192024-02-07T19:03:03.660Z 18 Acked 1017 Flush 0 Done Sent Done false
193202024-02-07T19:03:03.660Z 19 Acked 1018 Write 6 Done Sent Done false
193212024-02-07T19:03:03.660Z 20 Acked 1019 Write 6 Done Sent Done false
193222024-02-07T19:03:03.660Z 21 Acked 1020 Write 6 Done Sent Done false
193232024-02-07T19:03:03.660Z 22 Acked 1021 Write 10 Done Sent Done false
193242024-02-07T19:03:03.660Z 23 Acked 1022 Read 3 Done Sent Done false
193252024-02-07T19:03:03.660Z 24 Acked 1023 Read 7 Done Sent Done false
193262024-02-07T19:03:03.661Z 25 Acked 1024 Read 4 Done Sent Done false
193272024-02-07T19:03:03.661Z 26 Acked 1025 Read 10 Done Sent Done false
193282024-02-07T19:03:03.661Z 27 Acked 1026 Read 4 Done Sent Done false
193292024-02-07T19:03:03.661Z 28 Acked 1027 Read 1 Done Sent Done false
193302024-02-07T19:03:03.661Z 29 Acked 1028 Read 5 Done Sent Done false
193312024-02-07T19:03:03.661Z 30 Acked 1029 Read 5 Done Sent Done false
193322024-02-07T19:03:03.661Z STATES DS:0 DS:1 DS:2 TOTAL
193332024-02-07T19:03:03.661Z New 0 0 0 0
193342024-02-07T19:03:03.661Z Sent 0 26 0 26
193352024-02-07T19:03:03.661Z Done 30 4 30 64
193362024-02-07T19:03:03.661Z Skipped 0 0 0 0
193372024-02-07T19:03:03.661Z Error 0 0 0 0
193382024-02-07T19:03:03.661ZLast Flush: 1017 0 1017
193392024-02-07T19:03:03.661ZDownstairs last five completed:
193402024-02-07T19:03:03.661ZUpstairs last five completed: 30 29 28 27 26
193412024-02-07T19:03:03.661Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:02.853737864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1839}
193422024-02-07T19:03:03.661ZWrote out file "/tmp/test_repair_verify.data"
193432024-02-07T19:03:03.668Z
193442024-02-07T19:03:03.668ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193452024-02-07T19:03:03.684ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193462024-02-07T19:03:03.684Z 0 000-019 174 174 174 305 305 305 T F T <---
193472024-02-07T19:03:03.684Z 1 020-039 174 174 174 305 305 305 T F T <---
193482024-02-07T19:03:03.684Z 2 040-059 170 170 170 294 294 294 T F T <---
193492024-02-07T19:03:03.684Z 3 060-079 169 169 169 292 292 292 F F F
193502024-02-07T19:03:03.684Z 4 080-099 176 174 176 310 305 310 F F F <---
193512024-02-07T19:03:03.684Z 5 100-119 176 174 176 310 305 310 F F F <---
193522024-02-07T19:03:03.684Z 6 120-139 176 175 176 308 307 308 F T F <---
193532024-02-07T19:03:03.684Z 7 140-159 161 161 161 276 276 276 F F F
193542024-02-07T19:03:03.684Z 8 160-179 176 174 176 308 305 308 F F F <---
193552024-02-07T19:03:03.684Z 9 180-199 175 175 175 307 307 307 F F F
193562024-02-07T19:03:03.684Z 10 200-219 173 173 173 303 303 303 F F F
193572024-02-07T19:03:03.684Z 11 220-239 166 166 166 285 285 285 F F F
193582024-02-07T19:03:03.684Z 12 240-259 176 175 176 309 307 309 F F F <---
193592024-02-07T19:03:03.685Z 13 260-279 176 164 176 309 282 309 F F F <---
193602024-02-07T19:03:03.685Z 14 280-299 169 169 169 292 292 292 F F F
193612024-02-07T19:03:03.685Z 15 300-319 172 172 172 302 302 302 F F F
193622024-02-07T19:03:03.685Z 16 320-339 176 172 176 309 300 309 F F F <---
193632024-02-07T19:03:03.685Z 17 340-359 172 172 172 301 301 301 F F F
193642024-02-07T19:03:03.685Z 18 360-379 174 174 174 305 305 305 F F F
193652024-02-07T19:03:03.685Z 19 380-399 172 172 172 300 300 300 F F F
193662024-02-07T19:03:03.685Z 20 400-419 172 172 172 300 300 300 T F T <---
193672024-02-07T19:03:03.685Z 21 420-439 176 172 176 310 300 310 F F F <---
193682024-02-07T19:03:03.685Z 22 440-459 172 172 172 298 298 298 F F F
193692024-02-07T19:03:03.685Z 23 460-479 172 172 172 298 298 298 F F F
193702024-02-07T19:03:03.685Z 24 480-499 174 174 174 305 305 305 F F F
193712024-02-07T19:03:03.685Z 25 500-519 176 172 176 308 299 308 F F F <---
193722024-02-07T19:03:03.685Z 26 520-539 176 174 176 308 304 308 F F F <---
193732024-02-07T19:03:03.685Z 27 540-559 172 172 172 298 298 298 F F F
193742024-02-07T19:03:03.685Z 28 560-579 176 171 176 308 297 308 F F F <---
193752024-02-07T19:03:03.685Z 29 580-599 176 168 176 308 288 308 T T T <---
193762024-02-07T19:03:03.685ZMax gen: 176, Max flush: 310
193772024-02-07T19:03:03.690ZError: Difference in extent metadata found!
193782024-02-07T19:03:03.690ZA Difference in extent metadata is expected here
193792024-02-07T19:03:03.690ZOn loop 88
193802024-02-07T19:03:03.690Z
193812024-02-07T19:03:03.690ZVerifying data now
193822024-02-07T19:03:09.466ZNo change after verify
193832024-02-07T19:03:09.466ZLoop: 88 Downstairs dump after verify (and repair):
193842024-02-07T19:03:09.482ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193852024-02-07T19:03:09.482Z 0 000-019 177 177 177 311 311 311 F F F
193862024-02-07T19:03:09.482Z 1 020-039 177 177 177 311 311 311 F F F
193872024-02-07T19:03:09.482Z 2 040-059 177 177 177 311 311 311 F F F
193882024-02-07T19:03:09.483Z 3 060-079 169 169 169 292 292 292 F F F
193892024-02-07T19:03:09.483Z 4 080-099 176 176 176 310 310 310 F F F
193902024-02-07T19:03:09.483Z 5 100-119 176 176 176 310 310 310 F F F
193912024-02-07T19:03:09.483Z 6 120-139 176 176 176 308 308 308 F F F
193922024-02-07T19:03:09.483Z 7 140-159 161 161 161 276 276 276 F F F
193932024-02-07T19:03:09.484Z 8 160-179 176 176 176 308 308 308 F F F
193942024-02-07T19:03:09.484Z 9 180-199 175 175 175 307 307 307 F F F
193952024-02-07T19:03:09.492Z 10 200-219 173 173 173 303 303 303 F F F
193962024-02-07T19:03:09.493Z 11 220-239 166 166 166 285 285 285 F F F
193972024-02-07T19:03:09.493Z 12 240-259 176 176 176 309 309 309 F F F
193982024-02-07T19:03:09.493Z 13 260-279 176 176 176 309 309 309 F F F
193992024-02-07T19:03:09.493Z 14 280-299 169 169 169 292 292 292 F F F
194002024-02-07T19:03:09.493Z 15 300-319 172 172 172 302 302 302 F F F
194012024-02-07T19:03:09.493Z 16 320-339 176 176 176 309 309 309 F F F
194022024-02-07T19:03:09.493Z 17 340-359 172 172 172 301 301 301 F F F
194032024-02-07T19:03:09.493Z 18 360-379 174 174 174 305 305 305 F F F
194042024-02-07T19:03:09.493Z 19 380-399 172 172 172 300 300 300 F F F
194052024-02-07T19:03:09.493Z 20 400-419 177 177 177 311 311 311 F F F
194062024-02-07T19:03:09.493Z 21 420-439 176 176 176 310 310 310 F F F
194072024-02-07T19:03:09.493Z 22 440-459 172 172 172 298 298 298 F F F
194082024-02-07T19:03:09.493Z 23 460-479 172 172 172 298 298 298 F F F
194092024-02-07T19:03:09.493Z 24 480-499 174 174 174 305 305 305 F F F
194102024-02-07T19:03:09.493Z 25 500-519 176 176 176 308 308 308 F F F
194112024-02-07T19:03:09.493Z 26 520-539 176 176 176 308 308 308 F F F
194122024-02-07T19:03:09.493Z 27 540-559 172 172 172 298 298 298 F F F
194132024-02-07T19:03:09.493Z 28 560-579 176 176 176 308 308 308 F F F
194142024-02-07T19:03:09.493Z 29 580-599 177 177 177 311 311 311 F F F
194152024-02-07T19:03:09.494ZMax gen: 177, Max flush: 311
194162024-02-07T19:03:09.494Z
194172024-02-07T19:03:09.494ZBegin loop 89
194182024-02-07T19:03:09.494ZDownstairs to restart: 1
194192024-02-07T19:03:09.504Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.700752732Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847}
194202024-02-07T19:03:09.510Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701048102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847}
194212024-02-07T19:03:09.511ZCrucible runtime is spawned
194222024-02-07T19:03:09.511Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701113643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847}
194232024-02-07T19:03:09.511Z{"msg":"Crucible 484cf367-ce99-423c-a9e9-7aa36a9118c2 has session id: 24280001-6139-4ede-af7d-0f05983144a4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701155767Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194242024-02-07T19:03:09.511Z{"msg":"Upstairs opts: Upstairs UUID: 484cf367-ce99-423c-a9e9-7aa36a9118c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701197752Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194252024-02-07T19:03:09.511Z{"msg":"Crucible stats registered with UUID: 484cf367-ce99-423c-a9e9-7aa36a9118c2","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701227198Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194262024-02-07T19:03:09.511Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701245066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847}
194272024-02-07T19:03:09.511Z{"msg":"484cf367-ce99-423c-a9e9-7aa36a9118c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701260643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194282024-02-07T19:03:09.511Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701276671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194292024-02-07T19:03:09.511Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701295609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194302024-02-07T19:03:09.511Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701339303Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194312024-02-07T19:03:09.511Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701391936Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194322024-02-07T19:03:09.511Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701428161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194332024-02-07T19:03:09.512Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701482573Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194342024-02-07T19:03:09.512Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701533876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194352024-02-07T19:03:09.512Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701569551Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194362024-02-07T19:03:09.512Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701603657Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194372024-02-07T19:03:09.512Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:08.701640642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194382024-02-07T19:03:09.512Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:08.701676997Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194392024-02-07T19:03:09.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701723521Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194402024-02-07T19:03:09.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701751537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194412024-02-07T19:03:09.512Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:08.701788242Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194422024-02-07T19:03:09.512Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.70180158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194432024-02-07T19:03:09.512Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701815748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194442024-02-07T19:03:09.513Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.701957739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194452024-02-07T19:03:09.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.702336748Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194462024-02-07T19:03:09.513Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.702374383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194472024-02-07T19:03:09.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.70239765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194482024-02-07T19:03:09.513Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.702431565Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194492024-02-07T19:03:09.513Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.70298583Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194502024-02-07T19:03:09.513Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.70313129Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194512024-02-07T19:03:09.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.703544575Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194522024-02-07T19:03:09.513Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.70358159Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194532024-02-07T19:03:09.513Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.703597987Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194542024-02-07T19:03:09.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.703659859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194552024-02-07T19:03:09.513Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.703697534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194562024-02-07T19:03:09.513Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:08.703713622Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194572024-02-07T19:03:10.541Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.703596267Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194582024-02-07T19:03:10.541Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.703756105Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194592024-02-07T19:03:10.551Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.704631537Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194602024-02-07T19:03:10.551Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.704673321Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194612024-02-07T19:03:10.551Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705189871Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194622024-02-07T19:03:10.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705756065Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194632024-02-07T19:03:10.551Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705815817Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194642024-02-07T19:03:10.551Z{"msg":"[0]R flush_numbers[0..12]: [311, 311, 311, 292, 310, 310, 308, 276, 308, 307, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705846102Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194652024-02-07T19:03:10.552Z{"msg":"[0]R generation[0..12]: [177, 177, 177, 169, 176, 176, 176, 161, 176, 175, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705875418Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194662024-02-07T19:03:10.552Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705894046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194672024-02-07T19:03:10.552Z{"msg":"[1]R flush_numbers[0..12]: [311, 311, 311, 292, 310, 310, 308, 276, 308, 307, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705920532Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194682024-02-07T19:03:10.552Z{"msg":"[1]R generation[0..12]: [177, 177, 177, 169, 176, 176, 176, 161, 176, 175, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.70593939Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194692024-02-07T19:03:10.552ZWait for a query_work_queue command to finish before sending IO
194702024-02-07T19:03:10.552Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705969716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194712024-02-07T19:03:10.552Z{"msg":"[2]R flush_numbers[0..12]: [311, 311, 311, 292, 310, 310, 308, 276, 308, 307, 303, 285]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.705997232Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194722024-02-07T19:03:10.552Z{"msg":"[2]R generation[0..12]: [177, 177, 177, 169, 176, 176, 176, 161, 176, 175, 173, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706015839Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194732024-02-07T19:03:10.552Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706041866Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194742024-02-07T19:03:10.552Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706067243Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194752024-02-07T19:03:10.552Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706092449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194762024-02-07T19:03:10.552Z{"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706109857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194772024-02-07T19:03:10.552Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706153231Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194782024-02-07T19:03:10.553Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706175468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194792024-02-07T19:03:10.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706206764Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194802024-02-07T19:03:10.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706223881Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194812024-02-07T19:03:10.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706265336Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194822024-02-07T19:03:10.553Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706285193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194832024-02-07T19:03:10.553Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706316729Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194842024-02-07T19:03:10.553Z{"msg":"484cf367-ce99-423c-a9e9-7aa36a9118c2 is now active with session: 24280001-6139-4ede-af7d-0f05983144a4","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706334656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194852024-02-07T19:03:10.553Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706358013Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847,"session_id":"24280001-6139-4ede-af7d-0f05983144a4"}
194862024-02-07T19:03:10.553Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:09.706374431Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847}
194872024-02-07T19:03:10.553ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194882024-02-07T19:03:10.553ZLoading write count information from file "/tmp/test_repair_verify.data"
194892024-02-07T19:03:10.553ZRun Repair workload
194902024-02-07T19:03:10.554Z01/30 Write block 105 len 20480 data: 20 19 19 17 16
194912024-02-07T19:03:10.554Z02/30 Read block 105 len 4096
194922024-02-07T19:03:10.554Z03/30 Flush
194932024-02-07T19:03:10.554Z04/30 Write block 238 len 16384 data: 19 19 20 21
194942024-02-07T19:03:10.554Z05/30 Write block 543 len 20480 data: 10 9 9 11 16
194952024-02-07T19:03:10.554Z06/30 Flush
194962024-02-07T19:03:10.554Z07/30 Write block 255 len 24576 data: 12 10 7 9 11 12
194972024-02-07T19:03:10.554Z08/30 Read block 186 len 12288
194982024-02-07T19:03:10.554Z09/30 Flush
194992024-02-07T19:03:10.554Z10/30 Write block 399 len 20480 data: 16 16 13 14 10
195002024-02-07T19:03:10.554Z11/30 Write block 590 len 12288 data: 16 17 16
195012024-02-07T19:03:10.555Z12/30 Read block 102 len 20480
195022024-02-07T19:03:10.555Z13/30 Write block 294 len 16384 data: 13 13 12 13
195032024-02-07T19:03:10.555Z14/30 Write block 394 len 36864 data: 11 10 12 13 14 17 17 14 15
195042024-02-07T19:03:10.555Z15/30 Flush
195052024-02-07T19:03:10.555Z16/30 Write block 462 len 32768 data: 8 9 11 12 11 8 7 9
195062024-02-07T19:03:10.555Z17/30 Read block 356 len 16384
195072024-02-07T19:03:10.555Z18/30 Write block 141 len 24576 data: 11 11 13 13 13 15
195082024-02-07T19:03:10.555Z19/30 Read block 267 len 36864
195092024-02-07T19:03:10.555Z20/30 Write block 554 len 32768 data: 12 16 16 13 13 15 16 13
195102024-02-07T19:03:10.555Z21/30 Write block 350 len 40960 data: 16 15 11 10 15 12 9 5 7 9
195112024-02-07T19:03:10.555Z22/30 Read block 139 len 36864
195122024-02-07T19:03:10.555Z23/30 Read block 134 len 12288
195132024-02-07T19:03:10.555Z24/30 Read block 123 len 32768
195142024-02-07T19:03:10.556Z25/30 Write block 584 len 36864 data: 16 18 17 13 16 14 17 18 17
195152024-02-07T19:03:10.556Z26/30 Read block 353 len 28672
195162024-02-07T19:03:10.556Z27/30 Read block 284 len 40960
195172024-02-07T19:03:10.556Z28/30 Write block 340 len 16384 data: 16 12 12 13
195182024-02-07T19:03:10.556Z29/30 Read block 205 len 12288
195192024-02-07T19:03:10.556Z30/30 Write block 368 len 16384 data: 17 16 18 17
195202024-02-07T19:03:10.556Z----------------------------------------------------------------
195212024-02-07T19:03:10.556Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
195222024-02-07T19:03:10.556ZStates: Active Active Active
195232024-02-07T19:03:10.556ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195242024-02-07T19:03:10.556Z 1 Acked 1000 Write 5 Done Done Done false
195252024-02-07T19:03:10.556Z 2 Acked 1001 Read 1 Done Done Done false
195262024-02-07T19:03:10.556Z 3 Acked 1002 Flush 0 Done Sent Done false
195272024-02-07T19:03:10.557Z 4 Acked 1003 Write 4 Done Sent Done false
195282024-02-07T19:03:10.557Z 5 Acked 1004 Write 5 Done Sent Done false
195292024-02-07T19:03:10.557Z 6 Acked 1005 Flush 0 Done Sent Done false
195302024-02-07T19:03:10.557Z 7 Acked 1006 Write 6 Done Sent Done false
195312024-02-07T19:03:10.557Z 8 Acked 1007 Read 3 Done Sent Done false
195322024-02-07T19:03:10.557Z 9 Acked 1008 Flush 0 Done Sent Done false
195332024-02-07T19:03:10.557Z 10 Acked 1009 Write 5 Done Sent Done false
195342024-02-07T19:03:10.557Z 11 Acked 1010 Write 3 Done Sent Done false
195352024-02-07T19:03:10.557Z 12 Acked 1011 Read 5 Done Sent Done false
195362024-02-07T19:03:10.557Z 13 Acked 1012 Write 4 Done Sent Done false
195372024-02-07T19:03:10.557Z 14 Acked 1013 Write 9 Done Sent Done false
195382024-02-07T19:03:10.557Z 15 Acked 1014 Flush 0 Done Sent Done false
195392024-02-07T19:03:10.557Z 16 Acked 1015 Write 8 Done Sent Done false
195402024-02-07T19:03:10.557Z 17 Acked 1016 Read 4 Done Sent Done false
195412024-02-07T19:03:10.558Z 18 Acked 1017 Write 6 Done Sent Done false
195422024-02-07T19:03:10.558Z 19 Acked 1018 Read 9 Done Sent Done false
195432024-02-07T19:03:10.558Z 20 Acked 1019 Write 8 Done Sent Done false
195442024-02-07T19:03:10.558Z 21 Acked 1020 Write 10 Done Sent Done false
195452024-02-07T19:03:10.558Z 22 Acked 1021 Read 9 Done Sent Done false
195462024-02-07T19:03:10.558Z 23 Acked 1022 Read 3 Done Sent Done false
195472024-02-07T19:03:10.558Z 24 Acked 1023 Read 8 Done Sent Done false
195482024-02-07T19:03:10.558Z 25 Acked 1024 Write 9 Done Sent Done false
195492024-02-07T19:03:10.558Z 26 Acked 1025 Read 7 Done Sent Done false
195502024-02-07T19:03:10.558Z 27 Acked 1026 Read 10 Done Sent Done false
195512024-02-07T19:03:10.558Z 28 Acked 1027 Write 4 Done Sent Done false
195522024-02-07T19:03:10.558Z 29 Acked 1028 Read 3 Done Sent Done false
195532024-02-07T19:03:10.559Z 30 Acked 1029 Write 4 Sent Sent Sent false
195542024-02-07T19:03:10.559Z STATES DS:0 DS:1 DS:2 TOTAL
195552024-02-07T19:03:10.559Z New 0 0 0 0
195562024-02-07T19:03:10.559Z Sent 1 28 1 30
195572024-02-07T19:03:10.559Z Done 29 2 29 60
195582024-02-07T19:03:10.559Z Skipped 0 0 0 0
195592024-02-07T19:03:10.565Z Error 0 0 0 0
195602024-02-07T19:03:10.565ZLast Flush: 1014 0 1014
195612024-02-07T19:03:10.565ZDownstairs last five completed:
195622024-02-07T19:03:10.565ZUpstairs last five completed: 30 29 28 27 26
195632024-02-07T19:03:10.565Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:09.740029739Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1847}
195642024-02-07T19:03:10.565ZWrote out file "/tmp/test_repair_verify.data"
195652024-02-07T19:03:10.565Z
195662024-02-07T19:03:10.566ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
195672024-02-07T19:03:10.606ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195682024-02-07T19:03:10.606Z 0 000-019 177 177 177 311 311 311 F F F
195692024-02-07T19:03:10.606Z 1 020-039 177 177 177 311 311 311 F F F
195702024-02-07T19:03:10.606Z 2 040-059 177 177 177 311 311 311 F F F
195712024-02-07T19:03:10.606Z 3 060-079 169 169 169 292 292 292 F F F
195722024-02-07T19:03:10.606Z 4 080-099 176 176 176 310 310 310 F F F
195732024-02-07T19:03:10.606Z 5 100-119 178 176 178 312 310 312 F T F <---
195742024-02-07T19:03:10.606Z 6 120-139 176 176 176 308 308 308 F F F
195752024-02-07T19:03:10.607Z 7 140-159 161 161 161 276 276 276 T F T <---
195762024-02-07T19:03:10.607Z 8 160-179 176 176 176 308 308 308 F F F
195772024-02-07T19:03:10.607Z 9 180-199 175 175 175 307 307 307 F F F
195782024-02-07T19:03:10.607Z 10 200-219 173 173 173 303 303 303 F F F
195792024-02-07T19:03:10.607Z 11 220-239 178 166 178 313 285 313 F F F <---
195802024-02-07T19:03:10.607Z 12 240-259 178 176 178 314 309 314 F F F <---
195812024-02-07T19:03:10.607Z 13 260-279 178 176 178 314 309 314 F F F <---
195822024-02-07T19:03:10.616Z 14 280-299 178 169 178 315 292 315 F F F <---
195832024-02-07T19:03:10.617Z 15 300-319 172 172 172 302 302 302 F F F
195842024-02-07T19:03:10.617Z 16 320-339 176 176 176 309 309 309 F F F
195852024-02-07T19:03:10.617Z 17 340-359 172 172 172 301 301 301 T F T <---
195862024-02-07T19:03:10.617Z 18 360-379 174 174 174 305 305 305 T F T <---
195872024-02-07T19:03:10.617Z 19 380-399 178 172 178 315 300 315 F F F <---
195882024-02-07T19:03:10.617Z 20 400-419 178 177 178 315 311 315 F F F <---
195892024-02-07T19:03:10.618Z 21 420-439 176 176 176 310 310 310 F F F
195902024-02-07T19:03:10.618Z 22 440-459 172 172 172 298 298 298 F F F
195912024-02-07T19:03:10.618Z 23 460-479 172 172 172 298 298 298 T F T <---
195922024-02-07T19:03:10.618Z 24 480-499 174 174 174 305 305 305 F F F
195932024-02-07T19:03:10.618Z 25 500-519 176 176 176 308 308 308 F F F
195942024-02-07T19:03:10.618Z 26 520-539 176 176 176 308 308 308 F F F
195952024-02-07T19:03:10.618Z 27 540-559 178 172 178 313 298 313 T F T <---
195962024-02-07T19:03:10.618Z 28 560-579 176 176 176 308 308 308 T F T <---
195972024-02-07T19:03:10.618Z 29 580-599 178 177 178 315 311 315 T F T <---
195982024-02-07T19:03:10.618ZMax gen: 178, Max flush: 315
195992024-02-07T19:03:10.618ZError: Difference in extent metadata found!
196002024-02-07T19:03:10.618ZA Difference in extent metadata is expected here
196012024-02-07T19:03:10.618ZOn loop 89
196022024-02-07T19:03:10.618Z
196032024-02-07T19:03:10.618ZVerifying data now
196042024-02-07T19:03:15.251ZNo change after verify
196052024-02-07T19:03:15.251ZLoop: 89 Downstairs dump after verify (and repair):
196062024-02-07T19:03:15.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196072024-02-07T19:03:15.267Z 0 000-019 177 177 177 311 311 311 F F F
196082024-02-07T19:03:15.267Z 1 020-039 177 177 177 311 311 311 F F F
196092024-02-07T19:03:15.267Z 2 040-059 177 177 177 311 311 311 F F F
196102024-02-07T19:03:15.267Z 3 060-079 169 169 169 292 292 292 F F F
196112024-02-07T19:03:15.268Z 4 080-099 176 176 176 310 310 310 F F F
196122024-02-07T19:03:15.268Z 5 100-119 178 178 178 312 312 312 F F F
196132024-02-07T19:03:15.268Z 6 120-139 176 176 176 308 308 308 F F F
196142024-02-07T19:03:15.268Z 7 140-159 179 179 179 316 316 316 F F F
196152024-02-07T19:03:15.268Z 8 160-179 176 176 176 308 308 308 F F F
196162024-02-07T19:03:15.268Z 9 180-199 175 175 175 307 307 307 F F F
196172024-02-07T19:03:15.268Z 10 200-219 173 173 173 303 303 303 F F F
196182024-02-07T19:03:15.268Z 11 220-239 178 178 178 313 313 313 F F F
196192024-02-07T19:03:15.272Z 12 240-259 178 178 178 314 314 314 F F F
196202024-02-07T19:03:15.272Z 13 260-279 178 178 178 314 314 314 F F F
196212024-02-07T19:03:15.273Z 14 280-299 178 178 178 315 315 315 F F F
196222024-02-07T19:03:15.273Z 15 300-319 172 172 172 302 302 302 F F F
196232024-02-07T19:03:15.273Z 16 320-339 176 176 176 309 309 309 F F F
196242024-02-07T19:03:15.273Z 17 340-359 179 179 179 316 316 316 F F F
196252024-02-07T19:03:15.273Z 18 360-379 179 179 179 316 316 316 F F F
196262024-02-07T19:03:15.273Z 19 380-399 178 178 178 315 315 315 F F F
196272024-02-07T19:03:15.273Z 20 400-419 178 178 178 315 315 315 F F F
196282024-02-07T19:03:15.274Z 21 420-439 176 176 176 310 310 310 F F F
196292024-02-07T19:03:15.274Z 22 440-459 172 172 172 298 298 298 F F F
196302024-02-07T19:03:15.274Z 23 460-479 179 179 179 316 316 316 F F F
196312024-02-07T19:03:15.274Z 24 480-499 174 174 174 305 305 305 F F F
196322024-02-07T19:03:15.274Z 25 500-519 176 176 176 308 308 308 F F F
196332024-02-07T19:03:15.274Z 26 520-539 176 176 176 308 308 308 F F F
196342024-02-07T19:03:15.274Z 27 540-559 179 179 179 316 316 316 F F F
196352024-02-07T19:03:15.274Z 28 560-579 179 179 179 316 316 316 F F F
196362024-02-07T19:03:15.274Z 29 580-599 179 179 179 316 316 316 F F F
196372024-02-07T19:03:15.275ZMax gen: 179, Max flush: 316
196382024-02-07T19:03:15.275Z
196392024-02-07T19:03:15.275ZBegin loop 90
196402024-02-07T19:03:15.275ZDownstairs to restart: 0
196412024-02-07T19:03:15.292Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.483617726Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855}
196422024-02-07T19:03:15.301Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.483935463Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855}
196432024-02-07T19:03:15.301Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
196442024-02-07T19:03:15.301Z,"time":"2024-02-07T19:03:14.484040119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855}
196452024-02-07T19:03:15.301Z{"msg":"Crucible ecc070a1-b1c8-457c-927c-f13524f89453 has session id: c0420a2c-bcab-4f5c-80ee-03a3ca24d629","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484086313Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196462024-02-07T19:03:15.301Z{"msg":"Upstairs opts: Upstairs UUID: ecc070a1-b1c8-457c-927c-f13524f89453, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484134946Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196472024-02-07T19:03:15.301Z{"msg":"Crucible stats registered with UUID: ecc070a1-b1c8-457c-927c-f13524f89453","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484161673Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196482024-02-07T19:03:15.301Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484176731Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855}
196492024-02-07T19:03:15.301Z{"msg":"ecc070a1-b1c8-457c-927c-f13524f89453 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484205077Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196502024-02-07T19:03:15.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484222195Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196512024-02-07T19:03:15.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484259679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196522024-02-07T19:03:15.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484290615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196532024-02-07T19:03:15.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484317372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196542024-02-07T19:03:15.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484351927Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196552024-02-07T19:03:15.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484374584Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196562024-02-07T19:03:15.302Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484396531Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196572024-02-07T19:03:15.302Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484420898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196582024-02-07T19:03:15.302Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484457853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196592024-02-07T19:03:15.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484617651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196602024-02-07T19:03:15.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484669564Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196612024-02-07T19:03:15.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.484695831Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196622024-02-07T19:03:15.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.48536209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196632024-02-07T19:03:15.303Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.485454188Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196642024-02-07T19:03:15.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.485530087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196652024-02-07T19:03:15.303Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.485566903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196662024-02-07T19:03:15.303Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486035439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196672024-02-07T19:03:15.303Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486215895Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196682024-02-07T19:03:15.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486421707Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196692024-02-07T19:03:15.304Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486470071Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196702024-02-07T19:03:15.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486550449Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196712024-02-07T19:03:15.304Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486583805Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196722024-02-07T19:03:15.304Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486599293Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196732024-02-07T19:03:15.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486781078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196742024-02-07T19:03:15.304Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486813544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196752024-02-07T19:03:15.304Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.486829742Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196762024-02-07T19:03:15.304Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487035264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196772024-02-07T19:03:15.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487493892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196782024-02-07T19:03:15.305Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487538036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196792024-02-07T19:03:15.305Z{"msg":"[0]R flush_numbers[0..12]: [311, 311, 311, 292, 310, 312, 308, 316, 308, 307, 303, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487556193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196802024-02-07T19:03:15.305Z{"msg":"[0]R generation[0..12]: [177, 177, 177, 169, 176, 178, 176, 179, 176, 175, 173, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487578181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196812024-02-07T19:03:15.305Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487593268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196822024-02-07T19:03:15.305Z{"msg":"[1]R flush_numbers[0..12]: [311, 311, 311, 292, 310, 312, 308, 316, 308, 307, 303, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487614256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196832024-02-07T19:03:15.305Z{"msg":"[1]R generation[0..12]: [177, 177, 177, 169, 176, 178, 176, 179, 176, 175, 173, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487634943Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196842024-02-07T19:03:15.305ZWait for a query_work_queue command to finish before sending IO
196852024-02-07T19:03:15.305Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487675827Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196862024-02-07T19:03:15.305Z{"msg":"[2]R flush_numbers[0..12]: [311, 311, 311, 292, 310, 312, 308, 316, 308, 307, 303, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487712072Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196872024-02-07T19:03:15.306Z{"msg":"[2]R generation[0..12]: [177, 177, 177, 169, 176, 178, 176, 179, 176, 175, 173, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487735609Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196882024-02-07T19:03:15.306Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487750647Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196892024-02-07T19:03:15.306Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487775474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196902024-02-07T19:03:15.306Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487812289Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196912024-02-07T19:03:15.306Z{"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487844115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196922024-02-07T19:03:15.306Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487860902Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196932024-02-07T19:03:15.306Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487889468Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196942024-02-07T19:03:15.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487924454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196952024-02-07T19:03:15.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.48795476Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196962024-02-07T19:03:15.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.487969618Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196972024-02-07T19:03:15.307Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.488001513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196982024-02-07T19:03:15.307Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.488036119Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
196992024-02-07T19:03:15.307Z{"msg":"ecc070a1-b1c8-457c-927c-f13524f89453 is now active with session: c0420a2c-bcab-4f5c-80ee-03a3ca24d629","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.488060695Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
197002024-02-07T19:03:15.307Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.488075673Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855,"session_id":"c0420a2c-bcab-4f5c-80ee-03a3ca24d629"}
197012024-02-07T19:03:15.307Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:14.488108169Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855}
197022024-02-07T19:03:15.307ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197032024-02-07T19:03:15.307ZLoading write count information from file "/tmp/test_repair_verify.data"
197042024-02-07T19:03:15.307ZRun Repair workload
197052024-02-07T19:03:15.307Z01/30 Write block 423 len 40960 data: 11 12 17 14 12 17 15 14 15 14
197062024-02-07T19:03:15.308Z02/30 Read block 421 len 24576
197072024-02-07T19:03:15.312Z03/30 Read block 439 len 8192
197082024-02-07T19:03:15.313Z04/30 Write block 350 len 8192 data: 17 16
197092024-02-07T19:03:15.313Z05/30 Write block 32 len 32768 data: 21 20 22 20 20 15 14 11
197102024-02-07T19:03:15.313Z06/30 Read block 152 len 8192
197112024-02-07T19:03:15.313Z07/30 Write block 429 len 4096 data: 16
197122024-02-07T19:03:15.313Z08/30 Flush
197132024-02-07T19:03:15.313Z09/30 Write block 62 len 12288 data: 8 10 13
197142024-02-07T19:03:15.313Z10/30 Read block 147 len 20480
197152024-02-07T19:03:15.313Z11/30 Read block 73 len 28672
197162024-02-07T19:03:15.313Z12/30 Write block 467 len 20480 data: 9 8 10 11 10
197172024-02-07T19:03:15.313Z13/30 Flush
197182024-02-07T19:03:15.313Z14/30 Write block 216 len 20480 data: 7 7 8 8 9
197192024-02-07T19:03:15.314Z15/30 Write block 341 len 28672 data: 13 13 14 14 17 19 19
197202024-02-07T19:03:15.314Z16/30 Read block 70 len 12288
197212024-02-07T19:03:15.314Z17/30 Read block 41 len 20480
197222024-02-07T19:03:15.314Z18/30 Flush
197232024-02-07T19:03:15.314Z19/30 Write block 344 len 40960 data: 15 18 20 20 17 16 18 17 12 11
197242024-02-07T19:03:15.335Z20/30 Read block 321 len 40960
197252024-02-07T19:03:15.335Z21/30 Write block 211 len 12288 data: 11 11 12
197262024-02-07T19:03:15.335Z22/30 Read block 66 len 20480
197272024-02-07T19:03:15.335Z23/30 Flush
197282024-02-07T19:03:15.335Z24/30 Write block 94 len 36864 data: 14 17 19 18 16 15 13 13 12
197292024-02-07T19:03:15.335Z25/30 Read block 99 len 36864
197302024-02-07T19:03:15.335Z26/30 Write block 7 len 40960 data: 17 17 17 14 16 14 14 17 15 17
197312024-02-07T19:03:15.335Z27/30 Read block 207 len 12288
197322024-02-07T19:03:15.335Z28/30 Write block 515 len 40960 data: 20 18 16 18 16 16 14 9 8 10
197332024-02-07T19:03:15.335Z29/30 Write block 527 len 32768 data: 13 11 13 12 11 13 13 14
197342024-02-07T19:03:15.335Z30/30 Read block 287 len 28672
197352024-02-07T19:03:15.336Z----------------------------------------------------------------
197362024-02-07T19:03:15.336Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
197372024-02-07T19:03:15.336ZStates: Active Active Active
197382024-02-07T19:03:15.336ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197392024-02-07T19:03:15.336Z 1 Acked 1000 Write 10 Sent Done Done false
197402024-02-07T19:03:15.336Z 2 Acked 1001 Read 6 Sent Done Done false
197412024-02-07T19:03:15.336Z 3 Acked 1002 Read 2 Sent Done Done false
197422024-02-07T19:03:15.336Z 4 Acked 1003 Write 2 Sent Done Done false
197432024-02-07T19:03:15.336Z 5 Acked 1004 Write 8 Sent Done Done false
197442024-02-07T19:03:15.336Z 6 Acked 1005 Read 2 Sent Done Done false
197452024-02-07T19:03:15.336Z 7 Acked 1006 Write 1 Sent Done Done false
197462024-02-07T19:03:15.336Z 8 Acked 1007 Flush 0 Sent Done Done false
197472024-02-07T19:03:15.337Z 9 Acked 1008 Write 3 Sent Done Done false
197482024-02-07T19:03:15.337Z 10 Acked 1009 Read 5 Sent Done Done false
197492024-02-07T19:03:15.337Z 11 Acked 1010 Read 7 Sent Done Done false
197502024-02-07T19:03:15.337Z 12 Acked 1011 Write 5 Sent Done Done false
197512024-02-07T19:03:15.337Z 13 Acked 1012 Flush 0 Sent Done Done false
197522024-02-07T19:03:15.337Z 14 Acked 1013 Write 5 Sent Done Done false
197532024-02-07T19:03:15.337Z 15 Acked 1014 Write 7 Sent Done Done false
197542024-02-07T19:03:15.337Z 16 Acked 1015 Read 3 Sent Done Done false
197552024-02-07T19:03:15.337Z 17 Acked 1016 Read 5 Sent Done Done false
197562024-02-07T19:03:15.337Z 18 Acked 1017 Flush 0 Sent Done Done false
197572024-02-07T19:03:15.337Z 19 Acked 1018 Write 10 Sent Done Done false
197582024-02-07T19:03:15.337Z 20 Acked 1019 Read 10 Sent Done Done false
197592024-02-07T19:03:15.337Z 21 Acked 1020 Write 3 Sent Done Done false
197602024-02-07T19:03:15.337Z 22 Acked 1021 Read 5 Sent Done Done false
197612024-02-07T19:03:15.337Z 23 Acked 1022 Flush 0 Sent Done Done false
197622024-02-07T19:03:15.337Z 24 Acked 1023 Write 9 Sent Done Done false
197632024-02-07T19:03:15.337Z 25 Acked 1024 Read 9 Sent Done Done false
197642024-02-07T19:03:15.337Z 26 Acked 1025 Write 10 Sent Done Done false
197652024-02-07T19:03:15.337Z 27 Acked 1026 Read 3 Sent Done Done false
197662024-02-07T19:03:15.338Z 28 Acked 1027 Write 10 Sent Done Done false
197672024-02-07T19:03:15.338Z 29 Acked 1028 Write 8 Sent Done Done false
197682024-02-07T19:03:15.338Z 30 Acked 1029 Read 7 Sent Done Sent false
197692024-02-07T19:03:15.338Z STATES DS:0 DS:1 DS:2 TOTAL
197702024-02-07T19:03:15.338Z New 0 0 0 0
197712024-02-07T19:03:15.338Z Sent 30 0 1 31
197722024-02-07T19:03:15.338Z Done 0 30 29 59
197732024-02-07T19:03:15.338Z Skipped 0 0 0 0
197742024-02-07T19:03:15.338Z Error 0 0 0 0
197752024-02-07T19:03:15.338ZLast Flush: 0 1022 1022
197762024-02-07T19:03:15.338ZDownstairs last five completed:
197772024-02-07T19:03:15.338ZUpstairs last five completed: 30 29 28 27 26
197782024-02-07T19:03:15.338Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:14.523047543Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1855}
197792024-02-07T19:03:15.338ZWrote out file "/tmp/test_repair_verify.data"
197802024-02-07T19:03:15.338Z
197812024-02-07T19:03:15.343ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197822024-02-07T19:03:15.354ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197832024-02-07T19:03:15.354Z 0 000-019 177 177 177 311 311 311 F T T <---
197842024-02-07T19:03:15.354Z 1 020-039 177 180 180 311 317 317 F F F <---
197852024-02-07T19:03:15.354Z 2 040-059 177 177 177 311 311 311 F F F
197862024-02-07T19:03:15.354Z 3 060-079 169 180 180 292 318 318 F F F <---
197872024-02-07T19:03:15.354Z 4 080-099 176 176 176 310 310 310 F T T <---
197882024-02-07T19:03:15.354Z 5 100-119 178 178 178 312 312 312 F T T <---
197892024-02-07T19:03:15.354Z 6 120-139 176 176 176 308 308 308 F F F
197902024-02-07T19:03:15.354Z 7 140-159 179 179 179 316 316 316 F F F
197912024-02-07T19:03:15.354Z 8 160-179 176 176 176 308 308 308 F F F
197922024-02-07T19:03:15.355Z 9 180-199 175 175 175 307 307 307 F F F
197932024-02-07T19:03:15.355Z 10 200-219 173 180 180 303 320 320 F F F <---
197942024-02-07T19:03:15.358Z 11 220-239 178 180 180 313 319 319 F F F <---
197952024-02-07T19:03:15.358Z 12 240-259 178 178 178 314 314 314 F F F
197962024-02-07T19:03:15.359Z 13 260-279 178 178 178 314 314 314 F F F
197972024-02-07T19:03:15.359Z 14 280-299 178 178 178 315 315 315 F F F
197982024-02-07T19:03:15.359Z 15 300-319 172 172 172 302 302 302 F F F
197992024-02-07T19:03:15.359Z 16 320-339 176 176 176 309 309 309 F F F
198002024-02-07T19:03:15.359Z 17 340-359 179 180 180 316 320 320 F F F <---
198012024-02-07T19:03:15.359Z 18 360-379 179 179 179 316 316 316 F F F
198022024-02-07T19:03:15.359Z 19 380-399 178 178 178 315 315 315 F F F
198032024-02-07T19:03:15.359Z 20 400-419 178 178 178 315 315 315 F F F
198042024-02-07T19:03:15.359Z 21 420-439 176 180 180 310 317 317 F F F <---
198052024-02-07T19:03:15.359Z 22 440-459 172 172 172 298 298 298 F F F
198062024-02-07T19:03:15.359Z 23 460-479 179 180 180 316 318 318 F F F <---
198072024-02-07T19:03:15.360Z 24 480-499 174 174 174 305 305 305 F F F
198082024-02-07T19:03:15.360Z 25 500-519 176 176 176 308 308 308 F T T <---
198092024-02-07T19:03:15.360Z 26 520-539 176 176 176 308 308 308 F T T <---
198102024-02-07T19:03:15.360Z 27 540-559 179 179 179 316 316 316 F F F
198112024-02-07T19:03:15.360Z 28 560-579 179 179 179 316 316 316 F F F
198122024-02-07T19:03:15.360Z 29 580-599 179 179 179 316 316 316 F F F
198132024-02-07T19:03:15.360ZMax gen: 180, Max flush: 320
198142024-02-07T19:03:15.360ZError: Difference in extent metadata found!
198152024-02-07T19:03:15.360ZA Difference in extent metadata is expected here
198162024-02-07T19:03:15.360ZOn loop 90
198172024-02-07T19:03:15.360Z
198182024-02-07T19:03:15.364ZVerifying data now
198192024-02-07T19:03:19.952ZNo change after verify
198202024-02-07T19:03:19.953ZLoop: 90 Downstairs dump after verify (and repair):
198212024-02-07T19:03:19.969ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198222024-02-07T19:03:19.969Z 0 000-019 181 181 181 321 321 321 F F F
198232024-02-07T19:03:19.969Z 1 020-039 180 180 180 317 317 317 F F F
198242024-02-07T19:03:19.969Z 2 040-059 177 177 177 311 311 311 F F F
198252024-02-07T19:03:19.969Z 3 060-079 180 180 180 318 318 318 F F F
198262024-02-07T19:03:19.969Z 4 080-099 181 181 181 321 321 321 F F F
198272024-02-07T19:03:19.969Z 5 100-119 181 181 181 321 321 321 F F F
198282024-02-07T19:03:19.969Z 6 120-139 176 176 176 308 308 308 F F F
198292024-02-07T19:03:19.970Z 7 140-159 179 179 179 316 316 316 F F F
198302024-02-07T19:03:19.970Z 8 160-179 176 176 176 308 308 308 F F F
198312024-02-07T19:03:19.970Z 9 180-199 175 175 175 307 307 307 F F F
198322024-02-07T19:03:19.970Z 10 200-219 180 180 180 320 320 320 F F F
198332024-02-07T19:03:19.970Z 11 220-239 180 180 180 319 319 319 F F F
198342024-02-07T19:03:19.986Z 12 240-259 178 178 178 314 314 314 F F F
198352024-02-07T19:03:19.986Z 13 260-279 178 178 178 314 314 314 F F F
198362024-02-07T19:03:19.986Z 14 280-299 178 178 178 315 315 315 F F F
198372024-02-07T19:03:19.986Z 15 300-319 172 172 172 302 302 302 F F F
198382024-02-07T19:03:19.986Z 16 320-339 176 176 176 309 309 309 F F F
198392024-02-07T19:03:19.987Z 17 340-359 180 180 180 320 320 320 F F F
198402024-02-07T19:03:19.987Z 18 360-379 179 179 179 316 316 316 F F F
198412024-02-07T19:03:19.987Z 19 380-399 178 178 178 315 315 315 F F F
198422024-02-07T19:03:19.987Z 20 400-419 178 178 178 315 315 315 F F F
198432024-02-07T19:03:19.987Z 21 420-439 180 180 180 317 317 317 F F F
198442024-02-07T19:03:19.987Z 22 440-459 172 172 172 298 298 298 F F F
198452024-02-07T19:03:19.987Z 23 460-479 180 180 180 318 318 318 F F F
198462024-02-07T19:03:19.987Z 24 480-499 174 174 174 305 305 305 F F F
198472024-02-07T19:03:19.987Z 25 500-519 181 181 181 321 321 321 F F F
198482024-02-07T19:03:19.987Z 26 520-539 181 181 181 321 321 321 F F F
198492024-02-07T19:03:19.987Z 27 540-559 179 179 179 316 316 316 F F F
198502024-02-07T19:03:19.988Z 28 560-579 179 179 179 316 316 316 F F F
198512024-02-07T19:03:19.988Z 29 580-599 179 179 179 316 316 316 F F F
198522024-02-07T19:03:19.988ZMax gen: 181, Max flush: 321
198532024-02-07T19:03:19.988Z
198542024-02-07T19:03:19.988ZBegin loop 91
198552024-02-07T19:03:19.988ZDownstairs to restart: 1
198562024-02-07T19:03:19.994Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185295351Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863}
198572024-02-07T19:03:19.994Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
198582024-02-07T19:03:19.994Z,"time":"2024-02-07T19:03:19.18559378Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863}
198592024-02-07T19:03:19.994Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185644913Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863}
198602024-02-07T19:03:19.994Z{"msg":"Crucible 27d41fdb-e68d-4464-a408-3ee27622a558 has session id: 0422ad5f-4771-43d8-b770-d835e89cb168","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185674699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198612024-02-07T19:03:19.994Z{"msg":"Upstairs opts: Upstairs UUID: 27d41fdb-e68d-4464-a408-3ee27622a558, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185725453Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198622024-02-07T19:03:19.995Z{"msg":"Crucible stats registered with UUID: 27d41fdb-e68d-4464-a408-3ee27622a558","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185761068Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198632024-02-07T19:03:19.995Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185781325Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863}
198642024-02-07T19:03:19.995Z{"msg":"27d41fdb-e68d-4464-a408-3ee27622a558 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185935824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198652024-02-07T19:03:19.995Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185975129Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198662024-02-07T19:03:19.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.185994006Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198672024-02-07T19:03:19.995Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186020203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198682024-02-07T19:03:19.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186044669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198692024-02-07T19:03:19.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186067036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198702024-02-07T19:03:19.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186097192Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198712024-02-07T19:03:19.996Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186129168Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198722024-02-07T19:03:19.996Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186150285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198732024-02-07T19:03:19.996Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186190779Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198742024-02-07T19:03:19.996Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186365746Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198752024-02-07T19:03:19.996Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186417319Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198762024-02-07T19:03:19.996Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.186451284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198772024-02-07T19:03:19.996Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.187113035Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198782024-02-07T19:03:19.996Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.187153519Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198792024-02-07T19:03:19.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.187196643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198802024-02-07T19:03:19.997Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.18722268Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198812024-02-07T19:03:20.004Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.18773712Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198822024-02-07T19:03:20.004Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.187868832Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198832024-02-07T19:03:20.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188149585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198842024-02-07T19:03:20.004Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188197858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198852024-02-07T19:03:20.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188321471Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198862024-02-07T19:03:20.004Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188357246Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198872024-02-07T19:03:20.004Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188373754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198882024-02-07T19:03:20.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188433716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198892024-02-07T19:03:20.004Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188474701Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198902024-02-07T19:03:20.004Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188501257Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198912024-02-07T19:03:20.005Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.188722417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198922024-02-07T19:03:20.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189182715Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198932024-02-07T19:03:20.005Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.18922115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198942024-02-07T19:03:20.005Z{"msg":"[0]R flush_numbers[0..12]: [321, 317, 311, 318, 321, 321, 308, 316, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189237698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198952024-02-07T19:03:20.005Z{"msg":"[0]R generation[0..12]: [181, 180, 177, 180, 181, 181, 176, 179, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189259935Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198962024-02-07T19:03:20.005Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189280762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198972024-02-07T19:03:20.005Z{"msg":"[1]R flush_numbers[0..12]: [321, 317, 311, 318, 321, 321, 308, 316, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189304698Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198982024-02-07T19:03:20.006Z{"msg":"[1]R generation[0..12]: [181, 180, 177, 180, 181, 181, 176, 179, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189320186Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
198992024-02-07T19:03:20.006Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189341503Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199002024-02-07T19:03:20.006Z{"msg":"Wait for a query_work_queue command to finish before sending IO
199012024-02-07T19:03:20.006Z[2]R flush_numbers[0..12]: [321, 317, 311, 318, 321, 321, 308, 316, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189358451Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199022024-02-07T19:03:20.006Z{"msg":"[2]R generation[0..12]: [181, 180, 177, 180, 181, 181, 176, 179, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189380238Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199032024-02-07T19:03:20.006Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189395536Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199042024-02-07T19:03:20.006Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189415613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199052024-02-07T19:03:20.006Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189429632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199062024-02-07T19:03:20.006Z{"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189449029Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199072024-02-07T19:03:20.006Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189465887Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199082024-02-07T19:03:20.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189498892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199092024-02-07T19:03:20.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189520029Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199102024-02-07T19:03:20.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189545376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199112024-02-07T19:03:20.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189560184Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199122024-02-07T19:03:20.007Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.18958717Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199132024-02-07T19:03:20.007Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189623475Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199142024-02-07T19:03:20.007Z{"msg":"27d41fdb-e68d-4464-a408-3ee27622a558 is now active with session: 0422ad5f-4771-43d8-b770-d835e89cb168","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189671989Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199152024-02-07T19:03:20.007ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199162024-02-07T19:03:20.007Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.189709214Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863,"session_id":"0422ad5f-4771-43d8-b770-d835e89cb168"}
199172024-02-07T19:03:20.007Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:19.18974027Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863}
199182024-02-07T19:03:20.007ZLoading write count information from file "/tmp/test_repair_verify.data"
199192024-02-07T19:03:20.008ZRun Repair workload
199202024-02-07T19:03:20.008Z01/30 Write block 447 len 4096 data: 14
199212024-02-07T19:03:20.008Z02/30 Write block 409 len 16384 data: 11 14 16 13
199222024-02-07T19:03:20.008Z03/30 Read block 289 len 20480
199232024-02-07T19:03:20.008Z04/30 Write block 431 len 4096 data: 16
199242024-02-07T19:03:20.008Z05/30 Write block 133 len 28672 data: 13 13 10 7 6 4 7
199252024-02-07T19:03:20.008Z06/30 Read block 16 len 12288
199262024-02-07T19:03:20.008Z07/30 Write block 469 len 20480 data: 11 12 11 10 9
199272024-02-07T19:03:20.013Z08/30 Write block 82 len 8192 data: 11 10
199282024-02-07T19:03:20.013Z09/30 Read block 42 len 24576
199292024-02-07T19:03:20.013Z10/30 Write block 590 len 32768 data: 18 19 18 13 12 11 7 7
199302024-02-07T19:03:20.013Z11/30 Read block 433 len 8192
199312024-02-07T19:03:20.013Z12/30 Write block 457 len 24576 data: 15 16 15 14 13 9
199322024-02-07T19:03:20.014Z13/30 Write block 131 len 40960 data: 14 13 14 14 11 8 7 5 8 7
199332024-02-07T19:03:20.014Z14/30 Read block 179 len 12288
199342024-02-07T19:03:20.014Z15/30 Write block 400 len 8192 data: 18 15
199352024-02-07T19:03:20.014Z16/30 Flush
199362024-02-07T19:03:20.018Z17/30 Write block 50 len 28672 data: 10 13 11 12 14 16 16
199372024-02-07T19:03:20.018Z18/30 Read block 157 len 20480
199382024-02-07T19:03:20.019Z19/30 Write block 304 len 24576 data: 8 11 10 9 12 13
199392024-02-07T19:03:20.022Z20/30 Flush
199402024-02-07T19:03:20.022Z21/30 Write block 152 len 24576 data: 13 16 16 13 13 11
199412024-02-07T19:03:20.022Z22/30 Write block 343 len 12288 data: 15 16 19
199422024-02-07T19:03:20.023Z23/30 Read block 333 len 20480
199432024-02-07T19:03:20.026Z24/30 Write block 589 len 4096 data: 15
199442024-02-07T19:03:20.026Z25/30 Write block 544 len 40960 data: 10 10 12 17 18 19 18 15 13 13
199452024-02-07T19:03:20.026Z26/30 Write block 455 len 36864 data: 13 11 16 17 16 15 14 10 10
199462024-02-07T19:03:20.027Z27/30 Read block 202 len 20480
199472024-02-07T19:03:20.031Z28/30 Read block 477 len 24576
199482024-02-07T19:03:20.031Z29/30 Write block 412 len 4096 data: 14
199492024-02-07T19:03:20.031Z30/30 Read block 73 len 8192
199502024-02-07T19:03:20.031Z----------------------------------------------------------------
199512024-02-07T19:03:20.032Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
199522024-02-07T19:03:20.032ZStates: Active Active Active
199532024-02-07T19:03:20.032ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
199542024-02-07T19:03:20.032Z 1 Acked 1000 Write 1 Done Done Done false
199552024-02-07T19:03:20.032Z 2 Acked 1001 Write 4 Done Done Done false
199562024-02-07T19:03:20.032Z 3 Acked 1002 Read 5 Done Done Done false
199572024-02-07T19:03:20.032Z 4 Acked 1003 Write 1 Done Done Done false
199582024-02-07T19:03:20.032Z 5 Acked 1004 Write 7 Done Done Done false
199592024-02-07T19:03:20.032Z 6 Acked 1005 Read 3 Done Done Done false
199602024-02-07T19:03:20.033Z 7 Acked 1006 Write 5 Done Done Done false
199612024-02-07T19:03:20.033Z 8 Acked 1007 Write 2 Done Sent Done false
199622024-02-07T19:03:20.033Z 9 Acked 1008 Read 6 Done Sent Done false
199632024-02-07T19:03:20.033Z 10 Acked 1009 Write 8 Done Sent Done false
199642024-02-07T19:03:20.033Z 11 Acked 1010 Read 2 Done Sent Done false
199652024-02-07T19:03:20.033Z 12 Acked 1011 Write 6 Done Sent Done false
199662024-02-07T19:03:20.033Z 13 Acked 1012 Write 10 Done Sent Done false
199672024-02-07T19:03:20.033Z 14 Acked 1013 Read 3 Done Sent Done false
199682024-02-07T19:03:20.033Z 15 Acked 1014 Write 2 Done Sent Done false
199692024-02-07T19:03:20.034Z 16 Acked 1015 Flush 0 Done Sent Done false
199702024-02-07T19:03:20.034Z 17 Acked 1016 Write 7 Done Sent Done false
199712024-02-07T19:03:20.034Z 18 Acked 1017 Read 5 Done Sent Done false
199722024-02-07T19:03:20.034Z 19 Acked 1018 Write 6 Done Sent Done false
199732024-02-07T19:03:20.034Z 20 Acked 1019 Flush 0 Done Sent Done false
199742024-02-07T19:03:20.034Z 21 Acked 1020 Write 6 Done Sent Done false
199752024-02-07T19:03:20.034Z 22 Acked 1021 Write 3 Done Sent Done false
199762024-02-07T19:03:20.035Z 23 Acked 1022 Read 5 Done Sent Done false
199772024-02-07T19:03:20.035Z 24 Acked 1023 Write 1 Done Sent Done false
199782024-02-07T19:03:20.035Z 25 Acked 1024 Write 10 Done Sent Done false
199792024-02-07T19:03:20.035Z 26 Acked 1025 Write 9 Done Sent Done false
199802024-02-07T19:03:20.035Z 27 Acked 1026 Read 5 Done Sent Done false
199812024-02-07T19:03:20.035Z 28 Acked 1027 Read 6 Done Sent Done false
199822024-02-07T19:03:20.035Z 29 Acked 1028 Write 1 Done Sent Done false
199832024-02-07T19:03:20.035Z 30 Acked 1029 Read 2 Done Sent Sent false
199842024-02-07T19:03:20.035Z STATES DS:0 DS:1 DS:2 TOTAL
199852024-02-07T19:03:20.035Z New 0 0 0 0
199862024-02-07T19:03:20.036Z Sent 0 23 1 24
199872024-02-07T19:03:20.036Z Done 30 7 29 66
199882024-02-07T19:03:20.036Z Skipped 0 0 0 0
199892024-02-07T19:03:20.036Z Error 0 0 0 0
199902024-02-07T19:03:20.036ZLast Flush: 1019 0 1019
199912024-02-07T19:03:20.036ZDownstairs last five completed:
199922024-02-07T19:03:20.036ZUpstairs last five completed: 30 29 28 27 26
199932024-02-07T19:03:20.036Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:19.224625491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1863}
199942024-02-07T19:03:20.036ZWrote out file "/tmp/test_repair_verify.data"
199952024-02-07T19:03:20.040Z
199962024-02-07T19:03:20.041ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199972024-02-07T19:03:20.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199982024-02-07T19:03:20.057Z 0 000-019 181 181 181 321 321 321 F F F
199992024-02-07T19:03:20.057Z 1 020-039 180 180 180 317 317 317 F F F
200002024-02-07T19:03:20.057Z 2 040-059 182 177 182 323 311 323 F F F <---
200012024-02-07T19:03:20.057Z 3 060-079 180 180 180 318 318 318 F F F
200022024-02-07T19:03:20.058Z 4 080-099 182 181 182 322 321 322 F F F <---
200032024-02-07T19:03:20.058Z 5 100-119 181 181 181 321 321 321 F F F
200042024-02-07T19:03:20.062Z 6 120-139 182 176 182 322 308 322 F T F <---
200052024-02-07T19:03:20.062Z 7 140-159 182 179 182 322 316 322 T F T <---
200062024-02-07T19:03:20.062Z 8 160-179 176 176 176 308 308 308 F F F
200072024-02-07T19:03:20.062Z 9 180-199 175 175 175 307 307 307 F F F
200082024-02-07T19:03:20.062Z 10 200-219 180 180 180 320 320 320 F F F
200092024-02-07T19:03:20.062Z 11 220-239 180 180 180 319 319 319 F F F
200102024-02-07T19:03:20.062Z 12 240-259 178 178 178 314 314 314 F F F
200112024-02-07T19:03:20.062Z 13 260-279 178 178 178 314 314 314 F F F
200122024-02-07T19:03:20.063Z 14 280-299 178 178 178 315 315 315 F F F
200132024-02-07T19:03:20.063Z 15 300-319 182 172 182 323 302 323 F F F <---
200142024-02-07T19:03:20.063Z 16 320-339 176 176 176 309 309 309 F F F
200152024-02-07T19:03:20.063Z 17 340-359 180 180 180 320 320 320 T F T <---
200162024-02-07T19:03:20.063Z 18 360-379 179 179 179 316 316 316 F F F
200172024-02-07T19:03:20.063Z 19 380-399 178 178 178 315 315 315 F F F
200182024-02-07T19:03:20.063Z 20 400-419 182 178 182 322 315 322 T T T <---
200192024-02-07T19:03:20.063Z 21 420-439 182 180 182 322 317 322 F T F <---
200202024-02-07T19:03:20.063Z 22 440-459 182 172 182 322 298 322 T T T <---
200212024-02-07T19:03:20.063Z 23 460-479 182 180 182 322 318 322 T T T <---
200222024-02-07T19:03:20.063Z 24 480-499 174 174 174 305 305 305 F F F
200232024-02-07T19:03:20.064Z 25 500-519 181 181 181 321 321 321 F F F
200242024-02-07T19:03:20.064Z 26 520-539 181 181 181 321 321 321 F F F
200252024-02-07T19:03:20.064Z 27 540-559 179 179 179 316 316 316 T F T <---
200262024-02-07T19:03:20.064Z 28 560-579 179 179 179 316 316 316 F F F
200272024-02-07T19:03:20.064Z 29 580-599 182 179 182 322 316 322 T F T <---
200282024-02-07T19:03:20.064ZMax gen: 182, Max flush: 323
200292024-02-07T19:03:20.064ZError: Difference in extent metadata found!
200302024-02-07T19:03:20.064ZA Difference in extent metadata is expected here
200312024-02-07T19:03:20.064ZOn loop 91
200322024-02-07T19:03:20.064Z
200332024-02-07T19:03:20.068ZVerifying data now
200342024-02-07T19:03:25.665ZNo change after verify
200352024-02-07T19:03:25.665ZLoop: 91 Downstairs dump after verify (and repair):
200362024-02-07T19:03:25.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200372024-02-07T19:03:25.681Z 0 000-019 181 181 181 321 321 321 F F F
200382024-02-07T19:03:25.681Z 1 020-039 180 180 180 317 317 317 F F F
200392024-02-07T19:03:25.681Z 2 040-059 182 182 182 323 323 323 F F F
200402024-02-07T19:03:25.681Z 3 060-079 180 180 180 318 318 318 F F F
200412024-02-07T19:03:25.681Z 4 080-099 182 182 182 322 322 322 F F F
200422024-02-07T19:03:25.681Z 5 100-119 181 181 181 321 321 321 F F F
200432024-02-07T19:03:25.681Z 6 120-139 182 182 182 322 322 322 F F F
200442024-02-07T19:03:25.681Z 7 140-159 183 183 183 324 324 324 F F F
200452024-02-07T19:03:25.681Z 8 160-179 176 176 176 308 308 308 F F F
200462024-02-07T19:03:25.681Z 9 180-199 175 175 175 307 307 307 F F F
200472024-02-07T19:03:25.681Z 10 200-219 180 180 180 320 320 320 F F F
200482024-02-07T19:03:25.685Z 11 220-239 180 180 180 319 319 319 F F F
200492024-02-07T19:03:25.685Z 12 240-259 178 178 178 314 314 314 F F F
200502024-02-07T19:03:25.685Z 13 260-279 178 178 178 314 314 314 F F F
200512024-02-07T19:03:25.685Z 14 280-299 178 178 178 315 315 315 F F F
200522024-02-07T19:03:25.685Z 15 300-319 182 182 182 323 323 323 F F F
200532024-02-07T19:03:25.685Z 16 320-339 176 176 176 309 309 309 F F F
200542024-02-07T19:03:25.685Z 17 340-359 183 183 183 324 324 324 F F F
200552024-02-07T19:03:25.685Z 18 360-379 179 179 179 316 316 316 F F F
200562024-02-07T19:03:25.685Z 19 380-399 178 178 178 315 315 315 F F F
200572024-02-07T19:03:25.685Z 20 400-419 183 183 183 324 324 324 F F F
200582024-02-07T19:03:25.685Z 21 420-439 182 182 182 322 322 322 F F F
200592024-02-07T19:03:25.685Z 22 440-459 183 183 183 324 324 324 F F F
200602024-02-07T19:03:25.685Z 23 460-479 183 183 183 324 324 324 F F F
200612024-02-07T19:03:25.685Z 24 480-499 174 174 174 305 305 305 F F F
200622024-02-07T19:03:25.685Z 25 500-519 181 181 181 321 321 321 F F F
200632024-02-07T19:03:25.685Z 26 520-539 181 181 181 321 321 321 F F F
200642024-02-07T19:03:25.685Z 27 540-559 183 183 183 324 324 324 F F F
200652024-02-07T19:03:25.685Z 28 560-579 179 179 179 316 316 316 F F F
200662024-02-07T19:03:25.686Z 29 580-599 183 183 183 324 324 324 F F F
200672024-02-07T19:03:25.686ZMax gen: 183, Max flush: 324
200682024-02-07T19:03:25.686Z
200692024-02-07T19:03:25.686ZBegin loop 92
200702024-02-07T19:03:25.686ZDownstairs to restart: 2
200712024-02-07T19:03:25.703Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897261544Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871}
200722024-02-07T19:03:25.703Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897502362Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871}
200732024-02-07T19:03:25.703Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897536727Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871}
200742024-02-07T19:03:25.709Z{"msg":"Crucible 9e7c3bfd-3d89-41e2-91ca-2fd51076528d has session id: b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897566003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200752024-02-07T19:03:25.709Z{"msg":"Upstairs opts: Upstairs UUID: 9e7c3bfd-3d89-41e2-91ca-2fd51076528d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","Crucible runtime is spawned
200762024-02-07T19:03:25.709Zv":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897622345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200772024-02-07T19:03:25.709Z{"msg":"Crucible stats registered with UUID: 9e7c3bfd-3d89-41e2-91ca-2fd51076528d","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897647302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200782024-02-07T19:03:25.709Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897681477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871}
200792024-02-07T19:03:25.709Z{"msg":"9e7c3bfd-3d89-41e2-91ca-2fd51076528d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897789193Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200802024-02-07T19:03:25.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897817219Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200812024-02-07T19:03:25.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897832857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200822024-02-07T19:03:25.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897861803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200832024-02-07T19:03:25.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897880011Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200842024-02-07T19:03:25.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897903498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200852024-02-07T19:03:25.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897936203Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200862024-02-07T19:03:25.709Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.897978007Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200872024-02-07T19:03:25.709Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898003814Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200882024-02-07T19:03:25.709Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.89802867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200892024-02-07T19:03:25.709Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:24.898124197Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200902024-02-07T19:03:25.709Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:24.898234423Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200912024-02-07T19:03:25.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898262019Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200922024-02-07T19:03:25.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898301034Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200932024-02-07T19:03:25.710Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:24.89832754Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200942024-02-07T19:03:25.710Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898361076Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200952024-02-07T19:03:25.710Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.89839991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200962024-02-07T19:03:25.710Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898426867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200972024-02-07T19:03:25.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898863437Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200982024-02-07T19:03:25.710Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898901383Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
200992024-02-07T19:03:25.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898944367Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201002024-02-07T19:03:25.710Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.898965884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201012024-02-07T19:03:25.710Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.899455137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201022024-02-07T19:03:25.710Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.899496682Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201032024-02-07T19:03:25.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.900083083Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201042024-02-07T19:03:25.710Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.900120498Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201052024-02-07T19:03:25.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.900137525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201062024-02-07T19:03:25.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.900158812Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201072024-02-07T19:03:25.710Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.90017306Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201082024-02-07T19:03:25.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:24.900193388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201092024-02-07T19:03:26.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.899854843Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201102024-02-07T19:03:26.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.900009282Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201112024-02-07T19:03:26.712Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.901185903Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201122024-02-07T19:03:26.712Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.901221438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201132024-02-07T19:03:26.713Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.901742517Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201142024-02-07T19:03:26.713Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902335877Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201152024-02-07T19:03:26.713Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902376522Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201162024-02-07T19:03:26.713Z{"msg":"[0]R flush_numbers[0..12]: [321, 317, 323, 318, 322, 321, 322, 324, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902397049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201172024-02-07T19:03:26.714Z{"msg":"[0]R generation[0..12]: [181, 180, 182, 180, 182, 181, 182, 183, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902415776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201182024-02-07T19:03:26.714Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902444052Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201192024-02-07T19:03:26.714Z{"msg":"[1]R flush_numbers[0..12]: [321, 317, 323, 318, 322, 321, 322, 324, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.90246342Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201202024-02-07T19:03:26.714Z{"msg":"[1]R generation[0..12]: [181, 180, 182, 180, 182, 181, 182, 183, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902490096Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201212024-02-07T19:03:26.714Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902508864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201222024-02-07T19:03:26.714ZWait for a query_work_queue command to finish before sending IO
201232024-02-07T19:03:26.714Z{"msg":"[2]R flush_numbers[0..12]: [321, 317, 323, 318, 322, 321, 322, 324, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902541699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201242024-02-07T19:03:26.715Z{"msg":"[2]R generation[0..12]: [181, 180, 182, 180, 182, 181, 182, 183, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902568716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201252024-02-07T19:03:26.715Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902588283Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201262024-02-07T19:03:26.715Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902614289Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201272024-02-07T19:03:26.715Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902639926Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201282024-02-07T19:03:26.715Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902657264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201292024-02-07T19:03:26.715Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.90268337Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201302024-02-07T19:03:26.716Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902701838Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201312024-02-07T19:03:26.716Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902736033Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201322024-02-07T19:03:26.716Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.90276021Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201332024-02-07T19:03:26.716Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902782047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201342024-02-07T19:03:26.716Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902804593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201352024-02-07T19:03:26.716Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902825921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201362024-02-07T19:03:26.716Z{"msg":"9e7c3bfd-3d89-41e2-91ca-2fd51076528d is now active with session: b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902840009Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201372024-02-07T19:03:26.717Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902866855Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871,"session_id":"b9fa0e34-ce46-47b9-9b45-9f109c1b4ed9"}
201382024-02-07T19:03:26.717Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:25.902889232Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871}
201392024-02-07T19:03:26.717ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201402024-02-07T19:03:26.717ZLoading write count information from file "/tmp/test_repair_verify.data"
201412024-02-07T19:03:26.717ZRun Repair workload
201422024-02-07T19:03:26.722Z01/30 Write block 346 len 28672 data: 21 21 18 17 19 18 13
201432024-02-07T19:03:26.722Z02/30 Read block 532 len 8192
201442024-02-07T19:03:26.722Z03/30 Write block 100 len 12288 data: 14 14 13
201452024-02-07T19:03:26.723Z04/30 Read block 438 len 28672
201462024-02-07T19:03:26.723Z05/30 Read block 446 len 20480
201472024-02-07T19:03:26.723Z06/30 Read block 176 len 8192
201482024-02-07T19:03:26.723Z07/30 Read block 293 len 12288
201492024-02-07T19:03:26.723Z08/30 Read block 389 len 28672
201502024-02-07T19:03:26.723Z09/30 Read block 480 len 12288
201512024-02-07T19:03:26.724Z10/30 Read block 5 len 36864
201522024-02-07T19:03:26.724Z11/30 Read block 517 len 36864
201532024-02-07T19:03:26.724Z12/30 Read block 217 len 8192
201542024-02-07T19:03:26.724Z13/30 Read block 591 len 4096
201552024-02-07T19:03:26.724Z14/30 Write block 481 len 36864 data: 10 10 13 11 9 10 11 13 12
201562024-02-07T19:03:26.728Z15/30 Read block 114 len 24576
201572024-02-07T19:03:26.729Z16/30 Flush
201582024-02-07T19:03:26.729Z17/30 Write block 55 len 20480 data: 17 17 14 12 11
201592024-02-07T19:03:26.729Z18/30 Read block 574 len 32768
201602024-02-07T19:03:26.729Z19/30 Read block 165 len 16384
201612024-02-07T19:03:26.733Z20/30 Write block 535 len 36864 data: 17 17 18 16 15 16 17 16 11
201622024-02-07T19:03:26.734Z21/30 Read block 304 len 36864
201632024-02-07T19:03:26.734Z22/30 Read block 491 len 36864
201642024-02-07T19:03:26.734Z23/30 Flush
201652024-02-07T19:03:26.738Z24/30 Write block 289 len 8192 data: 19 21
201662024-02-07T19:03:26.738Z25/30 Read block 271 len 4096
201672024-02-07T19:03:26.738Z26/30 Read block 372 len 36864
201682024-02-07T19:03:26.739Z27/30 Read block 147 len 24576
201692024-02-07T19:03:26.739Z28/30 Read block 131 len 12288
201702024-02-07T19:03:26.739Z29/30 Read block 275 len 40960
201712024-02-07T19:03:26.744Z30/30 Read block 61 len 24576
201722024-02-07T19:03:26.744Z----------------------------------------------------------------
201732024-02-07T19:03:26.745Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
201742024-02-07T19:03:26.745ZStates: Active Active Active
201752024-02-07T19:03:26.745ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
201762024-02-07T19:03:26.745Z 1 Acked 1000 Write 7 Done Done Done false
201772024-02-07T19:03:26.745Z 2 Acked 1001 Read 2 Done Done Done false
201782024-02-07T19:03:26.745Z 3 Acked 1002 Write 3 Done Done Sent false
201792024-02-07T19:03:26.745Z 4 Acked 1003 Read 7 Done Done Sent false
201802024-02-07T19:03:26.745Z 5 Acked 1004 Read 5 Done Done Sent false
201812024-02-07T19:03:26.745Z 6 Acked 1005 Read 2 Done Done Sent false
201822024-02-07T19:03:26.746Z 7 Acked 1006 Read 3 Done Done Sent false
201832024-02-07T19:03:26.746Z 8 Acked 1007 Read 7 Done Done Sent false
201842024-02-07T19:03:26.746Z 9 Acked 1008 Read 3 Done Done Sent false
201852024-02-07T19:03:26.746Z 10 Acked 1009 Read 9 Done Done Sent false
201862024-02-07T19:03:26.746Z 11 Acked 1010 Read 9 Done Done Sent false
201872024-02-07T19:03:26.746Z 12 Acked 1011 Read 2 Done Done Sent false
201882024-02-07T19:03:26.746Z 13 Acked 1012 Read 1 Done Done Sent false
201892024-02-07T19:03:26.746Z 14 Acked 1013 Write 9 Done Done Sent false
201902024-02-07T19:03:26.746Z 15 Acked 1014 Read 6 Done Done Sent false
201912024-02-07T19:03:26.747Z 16 Acked 1015 Flush 0 Done Done Sent false
201922024-02-07T19:03:26.747Z 17 Acked 1016 Write 5 Done Done Sent false
201932024-02-07T19:03:26.747Z 18 Acked 1017 Read 8 Done Done Sent false
201942024-02-07T19:03:26.747Z 19 Acked 1018 Read 4 Done Done Sent false
201952024-02-07T19:03:26.747Z 20 Acked 1019 Write 9 Done Done Sent false
201962024-02-07T19:03:26.747Z 21 Acked 1020 Read 9 Done Done Sent false
201972024-02-07T19:03:26.747Z 22 Acked 1021 Read 9 Done Done Sent false
201982024-02-07T19:03:26.747Z 23 Acked 1022 Flush 0 Done Done Sent false
201992024-02-07T19:03:26.748Z 24 Acked 1023 Write 2 Done Done Sent false
202002024-02-07T19:03:26.748Z 25 Acked 1024 Read 1 Done Done Sent false
202012024-02-07T19:03:26.748Z 26 Acked 1025 Read 9 Done Done Sent false
202022024-02-07T19:03:26.749Z 27 Acked 1026 Read 6 Done Done Sent false
202032024-02-07T19:03:26.749Z 28 Acked 1027 Read 3 Done Done Sent false
202042024-02-07T19:03:26.749Z 29 Acked 1028 Read 10 Done Done Sent false
202052024-02-07T19:03:26.749Z 30 Acked 1029 Read 6 Done Done Sent false
202062024-02-07T19:03:26.749Z STATES DS:0 DS:1 DS:2 TOTAL
202072024-02-07T19:03:26.749Z New 0 0 0 0
202082024-02-07T19:03:26.749Z Sent 0 0 28 28
202092024-02-07T19:03:26.749Z Done 30 30 2 62
202102024-02-07T19:03:26.749Z Skipped 0 0 0 0
202112024-02-07T19:03:26.750Z Error 0 0 0 0
202122024-02-07T19:03:26.750ZLast Flush: 1022 1022 0
202132024-02-07T19:03:26.750ZDownstairs last five completed:
202142024-02-07T19:03:26.750ZUpstairs last five completed: 30 29 28 27 26
202152024-02-07T19:03:26.750Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:25.933602388Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1871}
202162024-02-07T19:03:26.750ZWrote out file "/tmp/test_repair_verify.data"
202172024-02-07T19:03:26.755Z
202182024-02-07T19:03:26.755ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202192024-02-07T19:03:26.764ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202202024-02-07T19:03:26.765Z 0 000-019 181 181 181 321 321 321 F F F
202212024-02-07T19:03:26.765Z 1 020-039 180 180 180 317 317 317 F F F
202222024-02-07T19:03:26.765Z 2 040-059 184 184 182 326 326 323 F F F <---
202232024-02-07T19:03:26.765Z 3 060-079 180 180 180 318 318 318 F F F
202242024-02-07T19:03:26.765Z 4 080-099 182 182 182 322 322 322 F F F
202252024-02-07T19:03:26.765Z 5 100-119 184 184 181 325 325 321 F F F <---
202262024-02-07T19:03:26.765Z 6 120-139 182 182 182 322 322 322 F F F
202272024-02-07T19:03:26.765Z 7 140-159 183 183 183 324 324 324 F F F
202282024-02-07T19:03:26.766Z 8 160-179 176 176 176 308 308 308 F F F
202292024-02-07T19:03:26.766Z 9 180-199 175 175 175 307 307 307 F F F
202302024-02-07T19:03:26.766Z 10 200-219 180 180 180 320 320 320 F F F
202312024-02-07T19:03:26.766Z 11 220-239 180 180 180 319 319 319 F F F
202322024-02-07T19:03:26.766Z 12 240-259 178 178 178 314 314 314 F F F
202332024-02-07T19:03:26.766Z 13 260-279 178 178 178 314 314 314 F F F
202342024-02-07T19:03:26.771Z 14 280-299 178 178 178 315 315 315 T T F <---
202352024-02-07T19:03:26.771Z 15 300-319 182 182 182 323 323 323 F F F
202362024-02-07T19:03:26.771Z 16 320-339 176 176 176 309 309 309 F F F
202372024-02-07T19:03:26.771Z 17 340-359 184 184 183 325 325 324 F F T <---
202382024-02-07T19:03:26.771Z 18 360-379 179 179 179 316 316 316 F F F
202392024-02-07T19:03:26.771Z 19 380-399 178 178 178 315 315 315 F F F
202402024-02-07T19:03:26.772Z 20 400-419 183 183 183 324 324 324 F F F
202412024-02-07T19:03:26.772Z 21 420-439 182 182 182 322 322 322 F F F
202422024-02-07T19:03:26.772Z 22 440-459 183 183 183 324 324 324 F F F
202432024-02-07T19:03:26.772Z 23 460-479 183 183 183 324 324 324 F F F
202442024-02-07T19:03:26.772Z 24 480-499 184 184 174 325 325 305 F F F <---
202452024-02-07T19:03:26.772Z 25 500-519 181 181 181 321 321 321 F F F
202462024-02-07T19:03:26.772Z 26 520-539 184 184 181 326 326 321 F F F <---
202472024-02-07T19:03:26.772Z 27 540-559 184 184 183 326 326 324 F F F <---
202482024-02-07T19:03:26.772Z 28 560-579 179 179 179 316 316 316 F F F
202492024-02-07T19:03:26.772Z 29 580-599 183 183 183 324 324 324 F F F
202502024-02-07T19:03:26.772ZMax gen: 184, Max flush: 326
202512024-02-07T19:03:26.772ZError: Difference in extent metadata found!
202522024-02-07T19:03:26.773ZA Difference in extent metadata is expected here
202532024-02-07T19:03:26.773ZOn loop 92
202542024-02-07T19:03:26.773Z
202552024-02-07T19:03:26.773ZVerifying data now
202562024-02-07T19:03:31.175ZNo change after verify
202572024-02-07T19:03:31.175ZLoop: 92 Downstairs dump after verify (and repair):
202582024-02-07T19:03:31.190ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202592024-02-07T19:03:31.190Z 0 000-019 181 181 181 321 321 321 F F F
202602024-02-07T19:03:31.190Z 1 020-039 180 180 180 317 317 317 F F F
202612024-02-07T19:03:31.190Z 2 040-059 184 184 184 326 326 326 F F F
202622024-02-07T19:03:31.190Z 3 060-079 180 180 180 318 318 318 F F F
202632024-02-07T19:03:31.190Z 4 080-099 182 182 182 322 322 322 F F F
202642024-02-07T19:03:31.191Z 5 100-119 184 184 184 325 325 325 F F F
202652024-02-07T19:03:31.191Z 6 120-139 182 182 182 322 322 322 F F F
202662024-02-07T19:03:31.191Z 7 140-159 183 183 183 324 324 324 F F F
202672024-02-07T19:03:31.191Z 8 160-179 176 176 176 308 308 308 F F F
202682024-02-07T19:03:31.191Z 9 180-199 175 175 175 307 307 307 F F F
202692024-02-07T19:03:31.191Z 10 200-219 180 180 180 320 320 320 F F F
202702024-02-07T19:03:31.192Z 11 220-239 180 180 180 319 319 319 F F F
202712024-02-07T19:03:31.192Z 12 240-259 178 178 178 314 314 314 F F F
202722024-02-07T19:03:31.196Z 13 260-279 178 178 178 314 314 314 F F F
202732024-02-07T19:03:31.197Z 14 280-299 185 185 185 327 327 327 F F F
202742024-02-07T19:03:31.197Z 15 300-319 182 182 182 323 323 323 F F F
202752024-02-07T19:03:31.197Z 16 320-339 176 176 176 309 309 309 F F F
202762024-02-07T19:03:31.197Z 17 340-359 184 184 184 325 325 325 F F F
202772024-02-07T19:03:31.197Z 18 360-379 179 179 179 316 316 316 F F F
202782024-02-07T19:03:31.197Z 19 380-399 178 178 178 315 315 315 F F F
202792024-02-07T19:03:31.197Z 20 400-419 183 183 183 324 324 324 F F F
202802024-02-07T19:03:31.197Z 21 420-439 182 182 182 322 322 322 F F F
202812024-02-07T19:03:31.197Z 22 440-459 183 183 183 324 324 324 F F F
202822024-02-07T19:03:31.198Z 23 460-479 183 183 183 324 324 324 F F F
202832024-02-07T19:03:31.198Z 24 480-499 184 184 184 325 325 325 F F F
202842024-02-07T19:03:31.198Z 25 500-519 181 181 181 321 321 321 F F F
202852024-02-07T19:03:31.198Z 26 520-539 184 184 184 326 326 326 F F F
202862024-02-07T19:03:31.198Z 27 540-559 184 184 184 326 326 326 F F F
202872024-02-07T19:03:31.198Z 28 560-579 179 179 179 316 316 316 F F F
202882024-02-07T19:03:31.198Z 29 580-599 183 183 183 324 324 324 F F F
202892024-02-07T19:03:31.199ZMax gen: 185, Max flush: 327
202902024-02-07T19:03:31.199Z
202912024-02-07T19:03:31.199ZBegin loop 93
202922024-02-07T19:03:31.199ZDownstairs to restart: 1
202932024-02-07T19:03:31.222Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.403385297Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879}
202942024-02-07T19:03:31.222Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.403723791Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879}
202952024-02-07T19:03:31.230Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
202962024-02-07T19:03:31.230Z,"time":"2024-02-07T19:03:30.403767235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879}
202972024-02-07T19:03:31.230Z{"msg":"Crucible b62c79a0-304a-473d-a1fe-864b40374450 has session id: 639150ac-7308-48cd-aebe-3e2ce8d07479","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.403807679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
202982024-02-07T19:03:31.230Z{"msg":"Upstairs opts: Upstairs UUID: b62c79a0-304a-473d-a1fe-864b40374450, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.403866491Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
202992024-02-07T19:03:31.231Z{"msg":"Crucible stats registered with UUID: b62c79a0-304a-473d-a1fe-864b40374450","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.403926983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203002024-02-07T19:03:31.231Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.40394723Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879}
203012024-02-07T19:03:31.231Z{"msg":"b62c79a0-304a-473d-a1fe-864b40374450 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404070964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203022024-02-07T19:03:31.231Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404103049Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203032024-02-07T19:03:31.231Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404119477Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203042024-02-07T19:03:31.231Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404133295Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203052024-02-07T19:03:31.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404164641Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203062024-02-07T19:03:31.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404193017Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203072024-02-07T19:03:31.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404235022Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203082024-02-07T19:03:31.232Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404284865Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203092024-02-07T19:03:31.232Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404313091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203102024-02-07T19:03:31.232Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404348776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203112024-02-07T19:03:31.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404450292Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203122024-02-07T19:03:31.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404621759Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203132024-02-07T19:03:31.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.404665123Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203142024-02-07T19:03:31.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.40513122Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203152024-02-07T19:03:31.233Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.405159237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203162024-02-07T19:03:31.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.405174415Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203172024-02-07T19:03:31.233Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.405224508Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203182024-02-07T19:03:31.233Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.405887668Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203192024-02-07T19:03:31.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.405977576Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203202024-02-07T19:03:31.233Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.40602114Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203212024-02-07T19:03:31.234Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406065074Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203222024-02-07T19:03:31.234Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406320669Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203232024-02-07T19:03:31.234Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406373012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203242024-02-07T19:03:31.234Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406435004Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203252024-02-07T19:03:31.234Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406521382Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203262024-02-07T19:03:31.234Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406559127Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203272024-02-07T19:03:31.234Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406575785Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203282024-02-07T19:03:31.235Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.406597002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203292024-02-07T19:03:31.235Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407037923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203302024-02-07T19:03:31.235Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407070578Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203312024-02-07T19:03:31.235Z{"msg":"[0]R flush_numbers[0..12]: [321, 317, 326, 318, 322, 325, 322, 324, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407086676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203322024-02-07T19:03:31.235Z{"msg":"[0]R generation[0..12]: [181, 180, 184, 180, 182, 184, 182, 183, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407108403Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203332024-02-07T19:03:31.235Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407123481Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203342024-02-07T19:03:31.235Z{"msg":"[1]R flush_numbers[0..12]: [321, 317, 326, 318, 322, 325, 322, 324, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407144688Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203352024-02-07T19:03:31.235Z{"msg":"[1]R generation[0..12]: [181, 180, 184, 180, 182, 184, 182, 183, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407159056Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203362024-02-07T19:03:31.236Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407189772Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
203372024-02-07T19:03:31.236Zdownstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203382024-02-07T19:03:31.236Z{"msg":"[2]R flush_numbers[0..12]: [321, 317, 326, 318, 322, 325, 322, 324, 308, 307, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407230137Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203392024-02-07T19:03:31.236Z{"msg":"[2]R generation[0..12]: [181, 180, 184, 180, 182, 184, 182, 183, 176, 175, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407264572Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203402024-02-07T19:03:31.236Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407297947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203412024-02-07T19:03:31.236Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407333822Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203422024-02-07T19:03:31.237Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407366208Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203432024-02-07T19:03:31.237Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407382236Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203442024-02-07T19:03:31.237Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407402513Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203452024-02-07T19:03:31.237Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407418911Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203462024-02-07T19:03:31.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407450676Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203472024-02-07T19:03:31.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407487092Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203482024-02-07T19:03:31.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407510639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203492024-02-07T19:03:31.238Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407524497Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203502024-02-07T19:03:31.238Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407552603Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203512024-02-07T19:03:31.238Z{"msg":"b62c79a0-304a-473d-a1fe-864b40374450 is now active with session: 639150ac-7308-48cd-aebe-3e2ce8d07479","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407589218Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203522024-02-07T19:03:31.238Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407614134Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879,"session_id":"639150ac-7308-48cd-aebe-3e2ce8d07479"}
203532024-02-07T19:03:31.238Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:30.407627593Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879}
203542024-02-07T19:03:31.238ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203552024-02-07T19:03:31.239ZLoading write count information from file "/tmp/test_repair_verify.data"
203562024-02-07T19:03:31.239ZRun Repair workload
203572024-02-07T19:03:31.239Z01/30 Write block 166 len 20480 data: 17 17 20 19 18
203582024-02-07T19:03:31.239Z02/30 Read block 518 len 24576
203592024-02-07T19:03:31.239Z03/30 Read block 327 len 20480
203602024-02-07T19:03:31.239Z04/30 Read block 551 len 20480
203612024-02-07T19:03:31.239Z05/30 Write block 254 len 36864 data: 16 13 11 8 10 12 13 11 14
203622024-02-07T19:03:31.240Z06/30 Write block 432 len 16384 data: 15 12 9 10
203632024-02-07T19:03:31.240Z07/30 Read block 389 len 36864
203642024-02-07T19:03:31.240Z08/30 Write block 198 len 40960 data: 16 15 13 13 14 11 13 14 11 11
203652024-02-07T19:03:31.240Z09/30 Write block 11 len 4096 data: 17
203662024-02-07T19:03:31.240Z10/30 Read block 491 len 28672
203672024-02-07T19:03:31.240Z11/30 Read block 379 len 32768
203682024-02-07T19:03:31.245Z12/30 Write block 519 len 20480 data: 17 17 15 10 9
203692024-02-07T19:03:31.245Z13/30 Write block 178 len 40960 data: 12 12 13 8 7 6 6 10 10 12
203702024-02-07T19:03:31.245Z14/30 Write block 192 len 16384 data: 16 14 16 12
203712024-02-07T19:03:31.245Z15/30 Write block 498 len 16384 data: 14 13 12 11
203722024-02-07T19:03:31.246Z16/30 Write block 105 len 28672 data: 21 20 20 18 17 12 13
203732024-02-07T19:03:31.246Z17/30 Flush
203742024-02-07T19:03:31.246Z18/30 Write block 228 len 20480 data: 11 13 12 11 15
203752024-02-07T19:03:31.246Z19/30 Write block 257 len 16384 data: 9 11 13 14
203762024-02-07T19:03:31.246Z20/30 Write block 233 len 20480 data: 15 15 22 21 22
203772024-02-07T19:03:31.246Z21/30 Read block 379 len 8192
203782024-02-07T19:03:31.247Z22/30 Write block 103 len 40960 data: 14 17 22 21 21 19 18 13 14 12
203792024-02-07T19:03:31.247Z23/30 Read block 41 len 28672
203802024-02-07T19:03:31.247Z24/30 Flush
203812024-02-07T19:03:31.247Z25/30 Write block 460 len 32768 data: 16 15 11 11 12 13 12 10
203822024-02-07T19:03:31.247Z26/30 Write block 478 len 24576 data: 12 11 11 11 11 14
203832024-02-07T19:03:31.248Z27/30 Write block 428 len 4096 data: 18
203842024-02-07T19:03:31.248Z28/30 Write block 273 len 20480 data: 11 9 11 10 10
203852024-02-07T19:03:31.248Z29/30 Write block 537 len 24576 data: 19 17 16 17 18 17
203862024-02-07T19:03:31.253Z30/30 Write block 76 len 12288 data: 14 15 18
203872024-02-07T19:03:31.253Z----------------------------------------------------------------
203882024-02-07T19:03:31.253Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
203892024-02-07T19:03:31.254ZStates: Active Active Active
203902024-02-07T19:03:31.254ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203912024-02-07T19:03:31.254Z 1 Acked 1000 Write 5 Done Sent Done false
203922024-02-07T19:03:31.254Z 2 Acked 1001 Read 6 Done Sent Done false
203932024-02-07T19:03:31.254Z 3 Acked 1002 Read 5 Done Sent Done false
203942024-02-07T19:03:31.254Z 4 Acked 1003 Read 5 Done Sent Done false
203952024-02-07T19:03:31.254Z 5 Acked 1004 Write 9 Done Sent Done false
203962024-02-07T19:03:31.255Z 6 Acked 1005 Write 4 Done Sent Done false
203972024-02-07T19:03:31.255Z 7 Acked 1006 Read 9 Done Sent Done false
203982024-02-07T19:03:31.255Z 8 Acked 1007 Write 10 Done Sent Done false
203992024-02-07T19:03:31.255Z 9 Acked 1008 Write 1 Done Sent Done false
204002024-02-07T19:03:31.255Z 10 Acked 1009 Read 7 Done Sent Done false
204012024-02-07T19:03:31.255Z 11 Acked 1010 Read 8 Done Sent Done false
204022024-02-07T19:03:31.255Z 12 Acked 1011 Write 5 Done Sent Done false
204032024-02-07T19:03:31.255Z 13 Acked 1012 Write 10 Done Sent Done false
204042024-02-07T19:03:31.256Z 14 Acked 1013 Write 4 Done Sent Done false
204052024-02-07T19:03:31.256Z 15 Acked 1014 Write 4 Done Sent Done false
204062024-02-07T19:03:31.256Z 16 Acked 1015 Write 7 Done Sent Done false
204072024-02-07T19:03:31.256Z 17 Acked 1016 Flush 0 Done Sent Done false
204082024-02-07T19:03:31.256Z 18 Acked 1017 Write 5 Done Sent Done false
204092024-02-07T19:03:31.256Z 19 Acked 1018 Write 4 Done Sent Done false
204102024-02-07T19:03:31.256Z 20 Acked 1019 Write 5 Done Sent Done false
204112024-02-07T19:03:31.256Z 21 Acked 1020 Read 2 Done Sent Done false
204122024-02-07T19:03:31.256Z 22 Acked 1021 Write 10 Done Sent Done false
204132024-02-07T19:03:31.257Z 23 Acked 1022 Read 7 Done Sent Done false
204142024-02-07T19:03:31.257Z 24 Acked 1023 Flush 0 Done Sent Done false
204152024-02-07T19:03:31.257Z 25 Acked 1024 Write 8 Done Sent Done false
204162024-02-07T19:03:31.257Z 26 Acked 1025 Write 6 Done Sent Done false
204172024-02-07T19:03:31.257Z 27 Acked 1026 Write 1 Done Sent Done false
204182024-02-07T19:03:31.257Z 28 Acked 1027 Write 5 Done Sent Done false
204192024-02-07T19:03:31.257Z 29 Acked 1028 Write 6 Sent Sent Sent false
204202024-02-07T19:03:31.257Z 30 Acked 1029 Write 3 Sent Sent Sent false
204212024-02-07T19:03:31.258Z STATES DS:0 DS:1 DS:2 TOTAL
204222024-02-07T19:03:31.258Z New 0 0 0 0
204232024-02-07T19:03:31.258Z Sent 2 30 2 34
204242024-02-07T19:03:31.258Z Done 28 0 28 56
204252024-02-07T19:03:31.258Z Skipped 0 0 0 0
204262024-02-07T19:03:31.258Z Error 0 0 0 0
204272024-02-07T19:03:31.258ZLast Flush: 1023 0 1023
204282024-02-07T19:03:31.258ZDownstairs last five completed:
204292024-02-07T19:03:31.259ZUpstairs last five completed: 30 29 28 27 26
204302024-02-07T19:03:31.259Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:30.438982012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1879}
204312024-02-07T19:03:31.259ZWrote out file "/tmp/test_repair_verify.data"
204322024-02-07T19:03:31.259Z
204332024-02-07T19:03:31.263ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204342024-02-07T19:03:31.271ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204352024-02-07T19:03:31.271Z 0 000-019 186 181 186 328 321 328 F F F <---
204362024-02-07T19:03:31.271Z 1 020-039 180 180 180 317 317 317 F F F
204372024-02-07T19:03:31.271Z 2 040-059 184 184 184 326 326 326 F F F
204382024-02-07T19:03:31.271Z 3 060-079 180 180 180 318 318 318 T F T <---
204392024-02-07T19:03:31.272Z 4 080-099 182 182 182 322 322 322 F F F
204402024-02-07T19:03:31.272Z 5 100-119 186 184 186 329 325 329 F F F <---
204412024-02-07T19:03:31.272Z 6 120-139 182 182 182 322 322 322 F F F
204422024-02-07T19:03:31.272Z 7 140-159 183 183 183 324 324 324 F F F
204432024-02-07T19:03:31.272Z 8 160-179 186 176 186 328 308 328 F F F <---
204442024-02-07T19:03:31.272Z 9 180-199 186 175 186 328 307 328 F F F <---
204452024-02-07T19:03:31.276Z 10 200-219 186 180 186 328 320 328 F F F <---
204462024-02-07T19:03:31.276Z 11 220-239 186 180 186 329 319 329 F F F <---
204472024-02-07T19:03:31.276Z 12 240-259 186 178 186 329 314 329 F F F <---
204482024-02-07T19:03:31.277Z 13 260-279 186 178 186 329 314 329 T F T <---
204492024-02-07T19:03:31.277Z 14 280-299 185 185 185 327 327 327 F F F
204502024-02-07T19:03:31.277Z 15 300-319 182 182 182 323 323 323 F F F
204512024-02-07T19:03:31.277Z 16 320-339 176 176 176 309 309 309 F F F
204522024-02-07T19:03:31.277Z 17 340-359 184 184 184 325 325 325 F F F
204532024-02-07T19:03:31.277Z 18 360-379 179 179 179 316 316 316 F F F
204542024-02-07T19:03:31.277Z 19 380-399 178 178 178 315 315 315 F F F
204552024-02-07T19:03:31.277Z 20 400-419 183 183 183 324 324 324 F F F
204562024-02-07T19:03:31.277Z 21 420-439 186 182 186 328 322 328 T F T <---
204572024-02-07T19:03:31.278Z 22 440-459 183 183 183 324 324 324 F F F
204582024-02-07T19:03:31.278Z 23 460-479 183 183 183 324 324 324 T F T <---
204592024-02-07T19:03:31.278Z 24 480-499 186 184 186 328 325 328 T F T <---
204602024-02-07T19:03:31.278Z 25 500-519 186 181 186 328 321 328 F F F <---
204612024-02-07T19:03:31.278Z 26 520-539 186 184 186 328 326 328 T F T <---
204622024-02-07T19:03:31.278Z 27 540-559 184 184 184 326 326 326 T F T <---
204632024-02-07T19:03:31.278Z 28 560-579 179 179 179 316 316 316 F F F
204642024-02-07T19:03:31.278Z 29 580-599 183 183 183 324 324 324 F F F
204652024-02-07T19:03:31.278ZMax gen: 186, Max flush: 329
204662024-02-07T19:03:31.279ZError: Difference in extent metadata found!
204672024-02-07T19:03:31.279ZA Difference in extent metadata is expected here
204682024-02-07T19:03:31.279ZOn loop 93
204692024-02-07T19:03:31.279Z
204702024-02-07T19:03:31.283ZVerifying data now
204712024-02-07T19:03:36.006ZNo change after verify
204722024-02-07T19:03:36.006ZLoop: 93 Downstairs dump after verify (and repair):
204732024-02-07T19:03:36.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204742024-02-07T19:03:36.022Z 0 000-019 186 186 186 328 328 328 F F F
204752024-02-07T19:03:36.022Z 1 020-039 180 180 180 317 317 317 F F F
204762024-02-07T19:03:36.023Z 2 040-059 184 184 184 326 326 326 F F F
204772024-02-07T19:03:36.023Z 3 060-079 187 187 187 330 330 330 F F F
204782024-02-07T19:03:36.023Z 4 080-099 182 182 182 322 322 322 F F F
204792024-02-07T19:03:36.023Z 5 100-119 186 186 186 329 329 329 F F F
204802024-02-07T19:03:36.023Z 6 120-139 182 182 182 322 322 322 F F F
204812024-02-07T19:03:36.023Z 7 140-159 183 183 183 324 324 324 F F F
204822024-02-07T19:03:36.023Z 8 160-179 186 186 186 328 328 328 F F F
204832024-02-07T19:03:36.027Z 9 180-199 186 186 186 328 328 328 F F F
204842024-02-07T19:03:36.028Z 10 200-219 186 186 186 328 328 328 F F F
204852024-02-07T19:03:36.028Z 11 220-239 186 186 186 329 329 329 F F F
204862024-02-07T19:03:36.028Z 12 240-259 186 186 186 329 329 329 F F F
204872024-02-07T19:03:36.028Z 13 260-279 187 187 187 330 330 330 F F F
204882024-02-07T19:03:36.028Z 14 280-299 185 185 185 327 327 327 F F F
204892024-02-07T19:03:36.028Z 15 300-319 182 182 182 323 323 323 F F F
204902024-02-07T19:03:36.028Z 16 320-339 176 176 176 309 309 309 F F F
204912024-02-07T19:03:36.029Z 17 340-359 184 184 184 325 325 325 F F F
204922024-02-07T19:03:36.029Z 18 360-379 179 179 179 316 316 316 F F F
204932024-02-07T19:03:36.029Z 19 380-399 178 178 178 315 315 315 F F F
204942024-02-07T19:03:36.029Z 20 400-419 183 183 183 324 324 324 F F F
204952024-02-07T19:03:36.030Z 21 420-439 187 187 187 330 330 330 F F F
204962024-02-07T19:03:36.030Z 22 440-459 183 183 183 324 324 324 F F F
204972024-02-07T19:03:36.030Z 23 460-479 187 187 187 330 330 330 F F F
204982024-02-07T19:03:36.033Z 24 480-499 187 187 187 330 330 330 F F F
204992024-02-07T19:03:36.033Z 25 500-519 186 186 186 328 328 328 F F F
205002024-02-07T19:03:36.033Z 26 520-539 187 187 187 330 330 330 F F F
205012024-02-07T19:03:36.033Z 27 540-559 187 187 187 330 330 330 F F F
205022024-02-07T19:03:36.034Z 28 560-579 179 179 179 316 316 316 F F F
205032024-02-07T19:03:36.034Z 29 580-599 183 183 183 324 324 324 F F F
205042024-02-07T19:03:36.034ZMax gen: 187, Max flush: 330
205052024-02-07T19:03:36.034Z
205062024-02-07T19:03:36.034ZBegin loop 94
205072024-02-07T19:03:36.034ZDownstairs to restart: 2
205082024-02-07T19:03:36.058Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235398324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887}
205092024-02-07T19:03:36.058Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.23564971Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887}
205102024-02-07T19:03:36.058Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235668197Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887}
205112024-02-07T19:03:36.058Z{"msg":"Crucible 2789bf51-48e8-4b90-83b2-0f9fc2235c81 has session id: 006f8003-6b3c-4d20-a2db-fbf7cd54fd61","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235690784Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205122024-02-07T19:03:36.058Z{"msg":"Upstairs opts: Upstairs UUID: 2789bf51-48e8-4b90-83b2-0f9fc2235c81, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
205132024-02-07T19:03:36.059Z,"time":"2024-02-07T19:03:35.235729229Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205142024-02-07T19:03:36.059Z{"msg":"Crucible stats registered with UUID: 2789bf51-48e8-4b90-83b2-0f9fc2235c81","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235761144Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205152024-02-07T19:03:36.059Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235835124Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887}
205162024-02-07T19:03:36.059Z{"msg":"2789bf51-48e8-4b90-83b2-0f9fc2235c81 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235920893Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205172024-02-07T19:03:36.059Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235947549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205182024-02-07T19:03:36.059Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.235962837Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205192024-02-07T19:03:36.059Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236018769Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205202024-02-07T19:03:36.059Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236049795Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205212024-02-07T19:03:36.059Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236073872Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205222024-02-07T19:03:36.059Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236122706Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205232024-02-07T19:03:36.059Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236144992Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205242024-02-07T19:03:36.059Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236169639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205252024-02-07T19:03:36.059Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236205894Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205262024-02-07T19:03:36.059Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:35.236242149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205272024-02-07T19:03:36.059Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:35.236282474Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205282024-02-07T19:03:36.059Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:35.236314749Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205292024-02-07T19:03:36.059Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236349225Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205302024-02-07T19:03:36.059Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.23638716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205312024-02-07T19:03:36.059Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236426364Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205322024-02-07T19:03:36.059Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.23645841Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205332024-02-07T19:03:36.059Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.236500824Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205342024-02-07T19:03:36.059Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.237021954Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205352024-02-07T19:03:36.060Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.237059699Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205362024-02-07T19:03:36.060Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.237083525Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205372024-02-07T19:03:36.060Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.237105313Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205382024-02-07T19:03:36.060Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.237660607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205392024-02-07T19:03:36.060Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.23779408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205402024-02-07T19:03:36.060Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.23815785Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205412024-02-07T19:03:36.060Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.238196015Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205422024-02-07T19:03:36.060Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.238213173Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205432024-02-07T19:03:36.060Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.238352034Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205442024-02-07T19:03:36.060Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.238390959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205452024-02-07T19:03:36.060Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:35.238408346Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205462024-02-07T19:03:37.046Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.237221067Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205472024-02-07T19:03:37.046Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.237453725Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205482024-02-07T19:03:37.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.238336776Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205492024-02-07T19:03:37.053Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.2383804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205502024-02-07T19:03:37.053Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.238903299Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205512024-02-07T19:03:37.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239515506Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205522024-02-07T19:03:37.053Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.23956801Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205532024-02-07T19:03:37.053Z{"msg":"[0]R flush_numbers[0..12]: [328, 317, 326, 330, 322, 329, 322, 324, 328, 328, 328, 329]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239593796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205542024-02-07T19:03:37.053Z{"msg":"[0]R generation[0..12]: [186, 180, 184, 187, 182, 186, 182, 183, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.23963439Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205552024-02-07T19:03:37.054Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239675555Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205562024-02-07T19:03:37.054Z{"msg":"[1]R flush_numbers[0..12]: [328, 317, 326, 330, 322, 329, 322, 324, 328, 328, 328, 329]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.23971597Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal",Wait for a query_work_queue command to finish before sending IO
205572024-02-07T19:03:37.054Z"pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205582024-02-07T19:03:37.054Z{"msg":"[1]R generation[0..12]: [186, 180, 184, 187, 182, 186, 182, 183, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239760024Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205592024-02-07T19:03:37.054Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239800178Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205602024-02-07T19:03:37.054Z{"msg":"[2]R flush_numbers[0..12]: [328, 317, 326, 330, 322, 329, 322, 324, 328, 328, 328, 329]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239840393Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205612024-02-07T19:03:37.054Z{"msg":"[2]R generation[0..12]: [186, 180, 184, 187, 182, 186, 182, 183, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239880237Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205622024-02-07T19:03:37.054Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.23993241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205632024-02-07T19:03:37.055Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.239972905Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205642024-02-07T19:03:37.055Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240001341Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205652024-02-07T19:03:37.055Z{"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240029947Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205662024-02-07T19:03:37.055Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240054044Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205672024-02-07T19:03:37.055Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.2400831Z","hostname":"ip-10-150-1-187.us-west-2.compute.internalRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
205682024-02-07T19:03:37.055Z","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205692024-02-07T19:03:37.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240114205Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205702024-02-07T19:03:37.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240141512Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205712024-02-07T19:03:37.056Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240182166Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205722024-02-07T19:03:37.056Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240214762Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205732024-02-07T19:03:37.056Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240252747Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205742024-02-07T19:03:37.056Z{"msg":"2789bf51-48e8-4b90-83b2-0f9fc2235c81 is now active with session: 006f8003-6b3c-4d20-a2db-fbf7cd54fd61","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240283613Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205752024-02-07T19:03:37.056Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240315478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887,"session_id":"006f8003-6b3c-4d20-a2db-fbf7cd54fd61"}
205762024-02-07T19:03:37.056Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:36.240343914Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887}
205772024-02-07T19:03:37.056ZLoading write count information from file "/tmp/test_repair_verify.data"
205782024-02-07T19:03:37.057ZRun Repair workload
205792024-02-07T19:03:37.057Z01/30 Write block 428 len 24576 data: 19 17 15 17 16 13
205802024-02-07T19:03:37.061Z02/30 Flush
205812024-02-07T19:03:37.061Z03/30 Write block 417 len 4096 data: 12
205822024-02-07T19:03:37.061Z04/30 Write block 75 len 28672 data: 9 15 16 19 17 15 11
205832024-02-07T19:03:37.061Z05/30 Read block 430 len 36864
205842024-02-07T19:03:37.061Z06/30 Write block 106 len 28672 data: 22 22 20 19 14 15 13
205852024-02-07T19:03:37.062Z07/30 Read block 391 len 16384
205862024-02-07T19:03:37.062Z08/30 Write block 367 len 12288 data: 16 18 17
205872024-02-07T19:03:37.062Z09/30 Read block 439 len 32768
205882024-02-07T19:03:37.062Z10/30 Read block 527 len 20480
205892024-02-07T19:03:37.062Z11/30 Read block 144 len 8192
205902024-02-07T19:03:37.062Z12/30 Write block 218 len 28672 data: 9 9 10 10 10 13 14
205912024-02-07T19:03:37.068Z13/30 Read block 202 len 40960
205922024-02-07T19:03:37.068Z14/30 Write block 377 len 24576 data: 15 15 12 12 10 9
205932024-02-07T19:03:37.069Z15/30 Read block 327 len 32768
205942024-02-07T19:03:37.069Z16/30 Read block 185 len 24576
205952024-02-07T19:03:37.069Z17/30 Read block 283 len 12288
205962024-02-07T19:03:37.074Z18/30 Flush
205972024-02-07T19:03:37.074Z19/30 Write block 579 len 8192 data: 11 13
205982024-02-07T19:03:37.074Z20/30 Read block 308 len 8192
205992024-02-07T19:03:37.074Z21/30 Read block 365 len 12288
206002024-02-07T19:03:37.074Z22/30 Read block 474 len 8192
206012024-02-07T19:03:37.075Z23/30 Read block 517 len 8192
206022024-02-07T19:03:37.075Z24/30 Write block 434 len 4096 data: 10
206032024-02-07T19:03:37.075Z25/30 Write block 74 len 40960 data: 9 10 16 17 20 18 16 12 12 11
206042024-02-07T19:03:37.090Z26/30 Write block 60 len 40960 data: 8 8 9 11 14 12 15 14 16 15
206052024-02-07T19:03:37.090Z27/30 Read block 367 len 24576
206062024-02-07T19:03:37.091Z28/30 Read block 500 len 16384
206072024-02-07T19:03:37.091Z29/30 Write block 185 len 4096 data: 11
206082024-02-07T19:03:37.091Z30/30 Read block 128 len 4096
206092024-02-07T19:03:37.091Z----------------------------------------------------------------
206102024-02-07T19:03:37.091Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:28
206112024-02-07T19:03:37.091ZStates: Active Active Active
206122024-02-07T19:03:37.091ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206132024-02-07T19:03:37.091Z 3 Acked 1002 Write 1 Done Done Done false
206142024-02-07T19:03:37.091Z 4 Acked 1003 Write 7 Done Done Sent false
206152024-02-07T19:03:37.091Z 5 Acked 1004 Read 9 Done Done Sent false
206162024-02-07T19:03:37.091Z 6 Acked 1005 Write 7 Done Done Sent false
206172024-02-07T19:03:37.091Z 7 Acked 1006 Read 4 Done Done Sent false
206182024-02-07T19:03:37.091Z 8 Acked 1007 Write 3 Done Done Sent false
206192024-02-07T19:03:37.091Z 9 Acked 1008 Read 8 Done Done Sent false
206202024-02-07T19:03:37.091Z 10 Acked 1009 Read 5 Done Done Sent false
206212024-02-07T19:03:37.091Z 11 Acked 1010 Read 2 Done Done Sent false
206222024-02-07T19:03:37.091Z 12 Acked 1011 Write 7 Done Done Sent false
206232024-02-07T19:03:37.091Z 13 Acked 1012 Read 10 Done Done Sent false
206242024-02-07T19:03:37.091Z 14 Acked 1013 Write 6 Done Done Sent false
206252024-02-07T19:03:37.091Z 15 Acked 1014 Read 8 Done Done Sent false
206262024-02-07T19:03:37.091Z 16 Acked 1015 Read 6 Done Done Sent false
206272024-02-07T19:03:37.091Z 17 Acked 1016 Read 3 Done Done Sent false
206282024-02-07T19:03:37.091Z 18 Acked 1017 Flush 0 Done Done Sent false
206292024-02-07T19:03:37.091Z 19 Acked 1018 Write 2 Done Done Sent false
206302024-02-07T19:03:37.092Z 20 Acked 1019 Read 2 Done Done Sent false
206312024-02-07T19:03:37.092Z 21 Acked 1020 Read 3 Done Done Sent false
206322024-02-07T19:03:37.092Z 22 Acked 1021 Read 2 Done Done Sent false
206332024-02-07T19:03:37.092Z 23 Acked 1022 Read 2 Done Done Sent false
206342024-02-07T19:03:37.092Z 24 Acked 1023 Write 1 Done Done Sent false
206352024-02-07T19:03:37.092Z 25 Acked 1024 Write 10 Done Done Sent false
206362024-02-07T19:03:37.092Z 26 Acked 1025 Write 10 Done Done Sent false
206372024-02-07T19:03:37.092Z 27 Acked 1026 Read 6 Done Done Sent false
206382024-02-07T19:03:37.092Z 28 Acked 1027 Read 4 Done Done Sent false
206392024-02-07T19:03:37.092Z 29 Acked 1028 Write 1 Done Done Sent false
206402024-02-07T19:03:37.092Z 30 Acked 1029 Read 1 Done Done Sent false
206412024-02-07T19:03:37.092Z STATES DS:0 DS:1 DS:2 TOTAL
206422024-02-07T19:03:37.092Z New 0 0 0 0
206432024-02-07T19:03:37.092Z Sent 0 0 27 27
206442024-02-07T19:03:37.092Z Done 28 28 1 57
206452024-02-07T19:03:37.092Z Skipped 0 0 0 0
206462024-02-07T19:03:37.092Z Error 0 0 0 0
206472024-02-07T19:03:37.092ZLast Flush: 1017 1017 1001
206482024-02-07T19:03:37.092ZDownstairs last five completed: 1001 1000
206492024-02-07T19:03:37.092ZUpstairs last five completed: 30 29 28 27 26
206502024-02-07T19:03:37.092Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:36.270403969Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1887}
206512024-02-07T19:03:37.092ZWrote out file "/tmp/test_repair_verify.data"
206522024-02-07T19:03:37.096Z
206532024-02-07T19:03:37.097ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206542024-02-07T19:03:37.104ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206552024-02-07T19:03:37.105Z 0 000-019 186 186 186 328 328 328 F F F
206562024-02-07T19:03:37.105Z 1 020-039 180 180 180 317 317 317 F F F
206572024-02-07T19:03:37.105Z 2 040-059 184 184 184 326 326 326 F F F
206582024-02-07T19:03:37.105Z 3 060-079 188 188 187 332 332 330 T T F <---
206592024-02-07T19:03:37.105Z 4 080-099 188 188 182 332 332 322 T T F <---
206602024-02-07T19:03:37.105Z 5 100-119 188 188 186 332 332 329 F F F <---
206612024-02-07T19:03:37.105Z 6 120-139 182 182 182 322 322 322 F F F
206622024-02-07T19:03:37.105Z 7 140-159 183 183 183 324 324 324 F F F
206632024-02-07T19:03:37.105Z 8 160-179 186 186 186 328 328 328 F F F
206642024-02-07T19:03:37.105Z 9 180-199 186 186 186 328 328 328 T T F <---
206652024-02-07T19:03:37.106Z 10 200-219 188 188 186 332 332 328 F F F <---
206662024-02-07T19:03:37.106Z 11 220-239 188 188 186 332 332 329 F F F <---
206672024-02-07T19:03:37.110Z 12 240-259 186 186 186 329 329 329 F F F
206682024-02-07T19:03:37.110Z 13 260-279 187 187 187 330 330 330 F F F
206692024-02-07T19:03:37.110Z 14 280-299 185 185 185 327 327 327 F F F
206702024-02-07T19:03:37.110Z 15 300-319 182 182 182 323 323 323 F F F
206712024-02-07T19:03:37.111Z 16 320-339 176 176 176 309 309 309 F F F
206722024-02-07T19:03:37.111Z 17 340-359 184 184 184 325 325 325 F F F
206732024-02-07T19:03:37.111Z 18 360-379 188 188 179 332 332 316 F F F <---
206742024-02-07T19:03:37.111Z 19 380-399 188 188 178 332 332 315 F F F <---
206752024-02-07T19:03:37.111Z 20 400-419 188 188 183 332 332 324 F F T <---
206762024-02-07T19:03:37.111Z 21 420-439 188 188 188 331 331 331 T T F <---
206772024-02-07T19:03:37.111Z 22 440-459 183 183 183 324 324 324 F F F
206782024-02-07T19:03:37.111Z 23 460-479 187 187 187 330 330 330 F F F
206792024-02-07T19:03:37.111Z 24 480-499 187 187 187 330 330 330 F F F
206802024-02-07T19:03:37.112Z 25 500-519 186 186 186 328 328 328 F F F
206812024-02-07T19:03:37.112Z 26 520-539 187 187 187 330 330 330 F F F
206822024-02-07T19:03:37.112Z 27 540-559 187 187 187 330 330 330 F F F
206832024-02-07T19:03:37.112Z 28 560-579 179 179 179 316 316 316 T T F <---
206842024-02-07T19:03:37.112Z 29 580-599 183 183 183 324 324 324 T T F <---
206852024-02-07T19:03:37.112ZMax gen: 188, Max flush: 332
206862024-02-07T19:03:37.112ZError: Difference in extent metadata found!
206872024-02-07T19:03:37.113ZA Difference in extent metadata is expected here
206882024-02-07T19:03:37.113ZOn loop 94
206892024-02-07T19:03:37.113Z
206902024-02-07T19:03:37.116ZVerifying data now
206912024-02-07T19:03:42.730ZNo change after verify
206922024-02-07T19:03:42.730ZLoop: 94 Downstairs dump after verify (and repair):
206932024-02-07T19:03:42.747ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206942024-02-07T19:03:42.747Z 0 000-019 186 186 186 328 328 328 F F F
206952024-02-07T19:03:42.747Z 1 020-039 180 180 180 317 317 317 F F F
206962024-02-07T19:03:42.747Z 2 040-059 184 184 184 326 326 326 F F F
206972024-02-07T19:03:42.747Z 3 060-079 189 189 189 333 333 333 F F F
206982024-02-07T19:03:42.747Z 4 080-099 189 189 189 333 333 333 F F F
206992024-02-07T19:03:42.747Z 5 100-119 188 188 188 332 332 332 F F F
207002024-02-07T19:03:42.747Z 6 120-139 182 182 182 322 322 322 F F F
207012024-02-07T19:03:42.747Z 7 140-159 183 183 183 324 324 324 F F F
207022024-02-07T19:03:42.747Z 8 160-179 186 186 186 328 328 328 F F F
207032024-02-07T19:03:42.747Z 9 180-199 189 189 189 333 333 333 F F F
207042024-02-07T19:03:42.747Z 10 200-219 188 188 188 332 332 332 F F F
207052024-02-07T19:03:42.748Z 11 220-239 188 188 188 332 332 332 F F F
207062024-02-07T19:03:42.751Z 12 240-259 186 186 186 329 329 329 F F F
207072024-02-07T19:03:42.751Z 13 260-279 187 187 187 330 330 330 F F F
207082024-02-07T19:03:42.751Z 14 280-299 185 185 185 327 327 327 F F F
207092024-02-07T19:03:42.751Z 15 300-319 182 182 182 323 323 323 F F F
207102024-02-07T19:03:42.752Z 16 320-339 176 176 176 309 309 309 F F F
207112024-02-07T19:03:42.752Z 17 340-359 184 184 184 325 325 325 F F F
207122024-02-07T19:03:42.752Z 18 360-379 188 188 188 332 332 332 F F F
207132024-02-07T19:03:42.752Z 19 380-399 188 188 188 332 332 332 F F F
207142024-02-07T19:03:42.752Z 20 400-419 188 188 188 332 332 332 F F F
207152024-02-07T19:03:42.752Z 21 420-439 189 189 189 333 333 333 F F F
207162024-02-07T19:03:42.752Z 22 440-459 183 183 183 324 324 324 F F F
207172024-02-07T19:03:42.752Z 23 460-479 187 187 187 330 330 330 F F F
207182024-02-07T19:03:42.752Z 24 480-499 187 187 187 330 330 330 F F F
207192024-02-07T19:03:42.752Z 25 500-519 186 186 186 328 328 328 F F F
207202024-02-07T19:03:42.752Z 26 520-539 187 187 187 330 330 330 F F F
207212024-02-07T19:03:42.752Z 27 540-559 187 187 187 330 330 330 F F F
207222024-02-07T19:03:42.753Z 28 560-579 189 189 189 333 333 333 F F F
207232024-02-07T19:03:42.753Z 29 580-599 189 189 189 333 333 333 F F F
207242024-02-07T19:03:42.753ZMax gen: 189, Max flush: 333
207252024-02-07T19:03:42.753Z
207262024-02-07T19:03:42.753ZBegin loop 95
207272024-02-07T19:03:42.753ZDownstairs to restart: 2
207282024-02-07T19:03:42.768Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.958973867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895}
207292024-02-07T19:03:42.768Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959221153Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895}
207302024-02-07T19:03:42.768Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959239651Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895}
207312024-02-07T19:03:42.768Z{"msg":"Crucible 34c31bc2-fd8f-4bfa-87dd-742419baa8c6 has session id: b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959256059Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207322024-02-07T19:03:42.787Z{"msg":"Upstairs opts: Upstairs UUID: 34c31bc2-fd8f-4bfa-87dd-742419baa8c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959311151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207332024-02-07T19:03:42.788Z{"msg":"Crucible stats registered with UUID: 34c31bc2-fd8f-4bfa-87dd-742419baa8c6","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959345167Z","hostname":"Crucible runtime is spawned
207342024-02-07T19:03:42.788Zip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207352024-02-07T19:03:42.788Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959445333Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895}
207362024-02-07T19:03:42.788Z{"msg":"34c31bc2-fd8f-4bfa-87dd-742419baa8c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959551629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207372024-02-07T19:03:42.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959602932Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207382024-02-07T19:03:42.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959633768Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207392024-02-07T19:03:42.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959674372Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207402024-02-07T19:03:42.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959703808Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207412024-02-07T19:03:42.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959748062Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207422024-02-07T19:03:42.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959816103Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207432024-02-07T19:03:42.790Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959856778Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207442024-02-07T19:03:42.790Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959883264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207452024-02-07T19:03:42.790Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959918909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207462024-02-07T19:03:42.791Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.959954384Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207472024-02-07T19:03:42.791Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.95998385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207482024-02-07T19:03:42.791Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.960034234Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207492024-02-07T19:03:42.791Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.960641921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207502024-02-07T19:03:42.792Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.960678656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207512024-02-07T19:03:42.792Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.960695434Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207522024-02-07T19:03:42.792Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.960733349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207532024-02-07T19:03:42.792Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961193997Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207542024-02-07T19:03:42.792Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961290964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207552024-02-07T19:03:42.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961709677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207562024-02-07T19:03:42.793Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961735923Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207572024-02-07T19:03:42.793Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961750571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207582024-02-07T19:03:42.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961798375Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207592024-02-07T19:03:42.794Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961841639Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207602024-02-07T19:03:42.794Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961857867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207612024-02-07T19:03:42.794Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961922038Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207622024-02-07T19:03:42.794Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.961949435Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207632024-02-07T19:03:42.795Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.962493171Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207642024-02-07T19:03:42.795Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.962934501Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207652024-02-07T19:03:42.795Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.962974406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207662024-02-07T19:03:42.795Z{"msg":"[0]R flush_numbers[0..12]: [328, 317, 326, 333, 333, 332, 322, 324, 328, 333, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.962991443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207672024-02-07T19:03:42.795Z{"msg":"[0]R generation[0..12]: [186, 180, 184, 189, 189, 188, 182, 183, 186, 189, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963014371Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207682024-02-07T19:03:42.795Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963029549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207692024-02-07T19:03:42.795Z{"msg":"[1]R flush_numbers[0..12]: [328, 317, 326, 333, 333, 332, 322, 324, 328, 333, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963050826Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207702024-02-07T19:03:42.795Z{"msg":"[1]R generation[0..12]: [186, 180, 184, 189, 189, 188, 182, 183, 186, 189, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963065464Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207712024-02-07T19:03:42.795Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963086961Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207722024-02-07T19:03:42.796Z{"msg":"[2]R flush_numbers[0..12]: [328, 317, 326, 333, 333, 332, 322, 324, 328, 333, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963101619Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207732024-02-07T19:03:42.796Z{"msg":"[2]R generation[0..12]: [186, 180, 184, 189, 189, 188, 182, 183, 186, 189, 188, 188]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
207742024-02-07T19:03:42.796Z,"time":"2024-02-07T19:03:41.963122486Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207752024-02-07T19:03:42.796Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963137324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207762024-02-07T19:03:42.796Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963158161Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207772024-02-07T19:03:42.796Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963172619Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207782024-02-07T19:03:42.796Z{"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963192176Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207792024-02-07T19:03:42.796Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963206005Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207802024-02-07T19:03:42.796Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963226072Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207812024-02-07T19:03:42.796Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.96323921Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207822024-02-07T19:03:42.796Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963265417Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207832024-02-07T19:03:42.796Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963279305Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207842024-02-07T19:03:42.796Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963293523Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207852024-02-07T19:03:42.804Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963324158Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207862024-02-07T19:03:42.804Z{"msg":"34c31bc2-fd8f-4bfa-87dd-742419baa8c6 is now active with session: b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963357264Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207872024-02-07T19:03:42.804Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.96339086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895,"session_id":"b10d3539-db5b-4e7f-8b29-2c0d6fc2bd95"}
207882024-02-07T19:03:42.804Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:41.963420036Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895}
207892024-02-07T19:03:42.805ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
207902024-02-07T19:03:42.805ZLoading write count information from file "/tmp/test_repair_verify.data"
207912024-02-07T19:03:42.805ZRun Repair workload
207922024-02-07T19:03:42.805Z01/30 Write block 330 len 40960 data: 11 15 14 17 18 17 17 15 17 20
207932024-02-07T19:03:42.805Z02/30 Write block 568 len 16384 data: 18 17 18 16
207942024-02-07T19:03:42.806Z03/30 Write block 436 len 24576 data: 11 14 10 13 14 14
207952024-02-07T19:03:42.806Z04/30 Read block 557 len 16384
207962024-02-07T19:03:42.806Z05/30 Write block 586 len 4096 data: 18
207972024-02-07T19:03:42.806Z06/30 Write block 25 len 16384 data: 17 14 14 15
207982024-02-07T19:03:42.806Z07/30 Read block 248 len 40960
207992024-02-07T19:03:42.806Z08/30 Read block 67 len 24576
208002024-02-07T19:03:42.806Z09/30 Read block 410 len 24576
208012024-02-07T19:03:42.807Z10/30 Read block 20 len 20480
208022024-02-07T19:03:42.807Z11/30 Read block 188 len 24576
208032024-02-07T19:03:42.807Z12/30 Write block 6 len 12288 data: 16 18 18
208042024-02-07T19:03:42.807Z13/30 Read block 281 len 40960
208052024-02-07T19:03:42.807Z14/30 Write block 463 len 40960 data: 12 13 14 13 11 9 12 13 12 11
208062024-02-07T19:03:42.808Z15/30 Read block 153 len 12288
208072024-02-07T19:03:42.808Z16/30 Write block 477 len 12288 data: 12 13 12
208082024-02-07T19:03:42.808Z17/30 Read block 393 len 28672
208092024-02-07T19:03:42.808Z18/30 Read block 498 len 24576
208102024-02-07T19:03:42.808Z19/30 Write block 443 len 40960 data: 15 16 17 14 15 13 10 12 13 12
208112024-02-07T19:03:42.808Z20/30 Read block 451 len 36864
208122024-02-07T19:03:42.808Z21/30 Read block 452 len 40960
208132024-02-07T19:03:42.808Z22/30 Read block 522 len 4096
208142024-02-07T19:03:42.808Z23/30 Write block 48 len 20480 data: 9 8 11 14 12
208152024-02-07T19:03:42.808Z24/30 Read block 351 len 20480
208162024-02-07T19:03:42.808Z25/30 Write block 415 len 28672 data: 10 7 13 14 11 12 12
208172024-02-07T19:03:42.808Z26/30 Read block 230 len 28672
208182024-02-07T19:03:42.808Z27/30 Read block 572 len 40960
208192024-02-07T19:03:42.808Z28/30 Write block 264 len 16384 data: 20 16 16 17
208202024-02-07T19:03:42.808Z29/30 Write block 18 len 32768 data: 17 20 21 21 19 21 18 18
208212024-02-07T19:03:42.809Z30/30 Read block 269 len 12288
208222024-02-07T19:03:42.809Z----------------------------------------------------------------
208232024-02-07T19:03:42.809Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
208242024-02-07T19:03:42.809ZStates: Active Active Active
208252024-02-07T19:03:42.809ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208262024-02-07T19:03:42.809Z 1 Acked 1000 Write 10 Done Done Sent false
208272024-02-07T19:03:42.809Z 2 Acked 1001 Write 4 Done Done Sent false
208282024-02-07T19:03:42.809Z 3 Acked 1002 Write 6 Done Done Sent false
208292024-02-07T19:03:42.809Z 4 Acked 1003 Read 4 Done Done Sent false
208302024-02-07T19:03:42.809Z 5 Acked 1004 Write 1 Done Done Sent false
208312024-02-07T19:03:42.809Z 6 Acked 1005 Write 4 Done Done Sent false
208322024-02-07T19:03:42.809Z 7 Acked 1006 Read 10 Done Done Sent false
208332024-02-07T19:03:42.809Z 8 Acked 1007 Read 6 Done Done Sent false
208342024-02-07T19:03:42.809Z 9 Acked 1008 Read 6 Done Done Sent false
208352024-02-07T19:03:42.809Z 10 Acked 1009 Read 5 Done Done Sent false
208362024-02-07T19:03:42.809Z 11 Acked 1010 Read 6 Done Done Sent false
208372024-02-07T19:03:42.809Z 12 Acked 1011 Write 3 Done Done Sent false
208382024-02-07T19:03:42.809Z 13 Acked 1012 Read 10 Done Done Sent false
208392024-02-07T19:03:42.809Z 14 Acked 1013 Write 10 Done Done Sent false
208402024-02-07T19:03:42.809Z 15 Acked 1014 Read 3 Done Done Sent false
208412024-02-07T19:03:42.809Z 16 Acked 1015 Write 3 Done Done Sent false
208422024-02-07T19:03:42.809Z 17 Acked 1016 Read 7 Done Done Sent false
208432024-02-07T19:03:42.810Z 18 Acked 1017 Read 6 Done Done Sent false
208442024-02-07T19:03:42.810Z 19 Acked 1018 Write 10 Done Done Sent false
208452024-02-07T19:03:42.810Z 20 Acked 1019 Read 9 Done Done Sent false
208462024-02-07T19:03:42.810Z 21 Acked 1020 Read 10 Done Done Sent false
208472024-02-07T19:03:42.810Z 22 Acked 1021 Read 1 Done Done Sent false
208482024-02-07T19:03:42.810Z 23 Acked 1022 Write 5 Done Done Sent false
208492024-02-07T19:03:42.810Z 24 Acked 1023 Read 5 Done Done Sent false
208502024-02-07T19:03:42.810Z 25 Acked 1024 Write 7 Done Done Sent false
208512024-02-07T19:03:42.810Z 26 Acked 1025 Read 7 Done Done Sent false
208522024-02-07T19:03:42.810Z 27 Acked 1026 Read 10 Done Done Sent false
208532024-02-07T19:03:42.810Z 28 Acked 1027 Write 4 Done Done Sent false
208542024-02-07T19:03:42.810Z 29 Acked 1028 Write 8 Done Done Sent false
208552024-02-07T19:03:42.810Z 30 Acked 1029 Read 3 Done Done Sent false
208562024-02-07T19:03:42.810Z STATES DS:0 DS:1 DS:2 TOTAL
208572024-02-07T19:03:42.810Z New 0 0 0 0
208582024-02-07T19:03:42.810Z Sent 0 0 30 30
208592024-02-07T19:03:42.810Z Done 30 30 0 60
208602024-02-07T19:03:42.810Z Skipped 0 0 0 0
208612024-02-07T19:03:42.810Z Error 0 0 0 0
208622024-02-07T19:03:42.810ZLast Flush: 0 0 0
208632024-02-07T19:03:42.810ZDownstairs last five completed:
208642024-02-07T19:03:42.810ZUpstairs last five completed: 30 29 28 27 26
208652024-02-07T19:03:42.810Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:41.992817629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1895}
208662024-02-07T19:03:42.810ZWrote out file "/tmp/test_repair_verify.data"
208672024-02-07T19:03:42.814Z
208682024-02-07T19:03:42.814ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
208692024-02-07T19:03:42.830ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208702024-02-07T19:03:42.830Z 0 000-019 186 186 186 328 328 328 T T F <---
208712024-02-07T19:03:42.830Z 1 020-039 180 180 180 317 317 317 T T F <---
208722024-02-07T19:03:42.830Z 2 040-059 184 184 184 326 326 326 T T F <---
208732024-02-07T19:03:42.830Z 3 060-079 189 189 189 333 333 333 F F F
208742024-02-07T19:03:42.830Z 4 080-099 189 189 189 333 333 333 F F F
208752024-02-07T19:03:42.830Z 5 100-119 188 188 188 332 332 332 F F F
208762024-02-07T19:03:42.830Z 6 120-139 182 182 182 322 322 322 F F F
208772024-02-07T19:03:42.830Z 7 140-159 183 183 183 324 324 324 F F F
208782024-02-07T19:03:42.831Z 8 160-179 186 186 186 328 328 328 F F F
208792024-02-07T19:03:42.831Z 9 180-199 189 189 189 333 333 333 F F F
208802024-02-07T19:03:42.831Z 10 200-219 188 188 188 332 332 332 F F F
208812024-02-07T19:03:42.837Z 11 220-239 188 188 188 332 332 332 F F F
208822024-02-07T19:03:42.837Z 12 240-259 186 186 186 329 329 329 F F F
208832024-02-07T19:03:42.837Z 13 260-279 187 187 187 330 330 330 T T F <---
208842024-02-07T19:03:42.837Z 14 280-299 185 185 185 327 327 327 F F F
208852024-02-07T19:03:42.837Z 15 300-319 182 182 182 323 323 323 F F F
208862024-02-07T19:03:42.837Z 16 320-339 176 176 176 309 309 309 T T F <---
208872024-02-07T19:03:42.838Z 17 340-359 184 184 184 325 325 325 F F F
208882024-02-07T19:03:42.838Z 18 360-379 188 188 188 332 332 332 F F F
208892024-02-07T19:03:42.838Z 19 380-399 188 188 188 332 332 332 F F F
208902024-02-07T19:03:42.838Z 20 400-419 188 188 188 332 332 332 T T F <---
208912024-02-07T19:03:42.838Z 21 420-439 189 189 189 333 333 333 T T F <---
208922024-02-07T19:03:42.838Z 22 440-459 183 183 183 324 324 324 T T F <---
208932024-02-07T19:03:42.838Z 23 460-479 187 187 187 330 330 330 T T F <---
208942024-02-07T19:03:42.838Z 24 480-499 187 187 187 330 330 330 F F F
208952024-02-07T19:03:42.838Z 25 500-519 186 186 186 328 328 328 F F F
208962024-02-07T19:03:42.838Z 26 520-539 187 187 187 330 330 330 F F F
208972024-02-07T19:03:42.838Z 27 540-559 187 187 187 330 330 330 F F F
208982024-02-07T19:03:42.838Z 28 560-579 189 189 189 333 333 333 T T F <---
208992024-02-07T19:03:42.838Z 29 580-599 189 189 189 333 333 333 T T F <---
209002024-02-07T19:03:42.839ZMax gen: 189, Max flush: 333
209012024-02-07T19:03:42.839ZError: Difference in extent metadata found!
209022024-02-07T19:03:42.839ZA Difference in extent metadata is expected here
209032024-02-07T19:03:42.839ZOn loop 95
209042024-02-07T19:03:42.839Z
209052024-02-07T19:03:42.848ZVerifying data now
209062024-02-07T19:03:47.407ZNo change after verify
209072024-02-07T19:03:47.407ZLoop: 95 Downstairs dump after verify (and repair):
209082024-02-07T19:03:47.422ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209092024-02-07T19:03:47.422Z 0 000-019 190 190 190 334 334 334 F F F
209102024-02-07T19:03:47.423Z 1 020-039 190 190 190 334 334 334 F F F
209112024-02-07T19:03:47.423Z 2 040-059 190 190 190 334 334 334 F F F
209122024-02-07T19:03:47.423Z 3 060-079 189 189 189 333 333 333 F F F
209132024-02-07T19:03:47.423Z 4 080-099 189 189 189 333 333 333 F F F
209142024-02-07T19:03:47.423Z 5 100-119 188 188 188 332 332 332 F F F
209152024-02-07T19:03:47.423Z 6 120-139 182 182 182 322 322 322 F F F
209162024-02-07T19:03:47.423Z 7 140-159 183 183 183 324 324 324 F F F
209172024-02-07T19:03:47.427Z 8 160-179 186 186 186 328 328 328 F F F
209182024-02-07T19:03:47.427Z 9 180-199 189 189 189 333 333 333 F F F
209192024-02-07T19:03:47.427Z 10 200-219 188 188 188 332 332 332 F F F
209202024-02-07T19:03:47.428Z 11 220-239 188 188 188 332 332 332 F F F
209212024-02-07T19:03:47.428Z 12 240-259 186 186 186 329 329 329 F F F
209222024-02-07T19:03:47.428Z 13 260-279 190 190 190 334 334 334 F F F
209232024-02-07T19:03:47.428Z 14 280-299 185 185 185 327 327 327 F F F
209242024-02-07T19:03:47.428Z 15 300-319 182 182 182 323 323 323 F F F
209252024-02-07T19:03:47.428Z 16 320-339 190 190 190 334 334 334 F F F
209262024-02-07T19:03:47.428Z 17 340-359 184 184 184 325 325 325 F F F
209272024-02-07T19:03:47.428Z 18 360-379 188 188 188 332 332 332 F F F
209282024-02-07T19:03:47.428Z 19 380-399 188 188 188 332 332 332 F F F
209292024-02-07T19:03:47.428Z 20 400-419 190 190 190 334 334 334 F F F
209302024-02-07T19:03:47.429Z 21 420-439 190 190 190 334 334 334 F F F
209312024-02-07T19:03:47.429Z 22 440-459 190 190 190 334 334 334 F F F
209322024-02-07T19:03:47.429Z 23 460-479 190 190 190 334 334 334 F F F
209332024-02-07T19:03:47.429Z 24 480-499 187 187 187 330 330 330 F F F
209342024-02-07T19:03:47.429Z 25 500-519 186 186 186 328 328 328 F F F
209352024-02-07T19:03:47.429Z 26 520-539 187 187 187 330 330 330 F F F
209362024-02-07T19:03:47.429Z 27 540-559 187 187 187 330 330 330 F F F
209372024-02-07T19:03:47.430Z 28 560-579 190 190 190 334 334 334 F F F
209382024-02-07T19:03:47.430Z 29 580-599 190 190 190 334 334 334 F F F
209392024-02-07T19:03:47.430ZMax gen: 190, Max flush: 334
209402024-02-07T19:03:47.430Z
209412024-02-07T19:03:47.430ZBegin loop 96
209422024-02-07T19:03:47.430ZDownstairs to restart: 1
209432024-02-07T19:03:47.444Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.634754115Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903}
209442024-02-07T19:03:47.445Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635043585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903}
209452024-02-07T19:03:47.445Z{"Crucible runtime is spawned
209462024-02-07T19:03:47.452Zmsg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635088849Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903}
209472024-02-07T19:03:47.452Z{"msg":"Crucible 0f917e9e-dad8-4098-b6b5-8d18a5f67659 has session id: 5220247d-6005-43a4-8d15-a4ba88bc8d51","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635123615Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209482024-02-07T19:03:47.452Z{"msg":"Upstairs opts: Upstairs UUID: 0f917e9e-dad8-4098-b6b5-8d18a5f67659, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.63515835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209492024-02-07T19:03:47.452Z{"msg":"Crucible stats registered with UUID: 0f917e9e-dad8-4098-b6b5-8d18a5f67659","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635175638Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209502024-02-07T19:03:47.452Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635208763Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903}
209512024-02-07T19:03:47.453Z{"msg":"0f917e9e-dad8-4098-b6b5-8d18a5f67659 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635298421Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209522024-02-07T19:03:47.453Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635323148Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209532024-02-07T19:03:47.453Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635338256Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209542024-02-07T19:03:47.453Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635366952Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209552024-02-07T19:03:47.453Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635389299Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209562024-02-07T19:03:47.453Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635413745Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209572024-02-07T19:03:47.453Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635443681Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209582024-02-07T19:03:47.453Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635466118Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209592024-02-07T19:03:47.453Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635488355Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209602024-02-07T19:03:47.453Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635511162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209612024-02-07T19:03:47.453Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635629426Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209622024-02-07T19:03:47.453Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635689858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209632024-02-07T19:03:47.453Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.635720284Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209642024-02-07T19:03:47.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.63641857Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209652024-02-07T19:03:47.454Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.636454854Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209662024-02-07T19:03:47.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.636470892Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209672024-02-07T19:03:47.454Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.636497909Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209682024-02-07T19:03:47.454Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637105047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209692024-02-07T19:03:47.454Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.63715206Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209702024-02-07T19:03:47.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637502833Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209712024-02-07T19:03:47.454Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637551856Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209722024-02-07T19:03:47.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.63774385Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209732024-02-07T19:03:47.454Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637782345Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209742024-02-07T19:03:47.454Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637809142Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209752024-02-07T19:03:47.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637849356Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209762024-02-07T19:03:47.454Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637885211Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209772024-02-07T19:03:47.455Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.637914087Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209782024-02-07T19:03:47.455Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638082365Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209792024-02-07T19:03:47.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638528204Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209802024-02-07T19:03:47.455Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638568149Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209812024-02-07T19:03:47.455Z{"msg":"[0]R flush_numbers[0..12]: [334, 334, 334, 333, 333, 332, 322, 324, 328, 333, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638585386Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209822024-02-07T19:03:47.455Z{"msg":"[0]R generation[0..12]: [190, 190, 190, 189, 189, 188, 182, 183, 186, 189, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638607183Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209832024-02-07T19:03:47.455Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638622152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209842024-02-07T19:03:47.455Z{"msg":"[1]R flush_numbers[0..12]: [334, 334, 334, 333, 333, 332, 322, 324, 328, 333, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638643729Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209852024-02-07T19:03:47.455Z{"msg":"[1]R generation[0..12]: [190, 190, 190, 189, 189, 188, 182, 183, 186, 189, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638658227Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209862024-02-07T19:03:47.455Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638684933Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209872024-02-07T19:03:47.455Z{"msg":"[2]R flush_numbers[0..12]: [334, 334, 334, 333, 333, 332, 322, 324, 328, 333, 332, 332]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
209882024-02-07T19:03:47.455Z2024-02-07T19:03:46.6387066Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209892024-02-07T19:03:47.455Z{"msg":"[2]R generation[0..12]: [190, 190, 190, 189, 189, 188, 182, 183, 186, 189, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638734836Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209902024-02-07T19:03:47.456Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638756803Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209912024-02-07T19:03:47.456Z{"msg":"Max found gen is 191","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638771141Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209922024-02-07T19:03:47.456Z{"msg":"Generation requested: 192 >= found:191","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638797588Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209932024-02-07T19:03:47.456Z{"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638821585Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209942024-02-07T19:03:47.456Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638858889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209952024-02-07T19:03:47.456Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638888815Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209962024-02-07T19:03:47.456Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638902174Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209972024-02-07T19:03:47.456Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638934619Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209982024-02-07T19:03:47.456Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638970574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
209992024-02-07T19:03:47.456Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.638997231Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
210002024-02-07T19:03:47.456Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.639011309Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
210012024-02-07T19:03:47.456Z{"msg":"0f917e9e-dad8-4098-b6b5-8d18a5f67659 is now active with session: 5220247d-6005-43a4-8d15-a4ba88bc8d51","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.639045454Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
210022024-02-07T19:03:47.457Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.63907878Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903,"session_id":"5220247d-6005-43a4-8d15-a4ba88bc8d51"}
210032024-02-07T19:03:47.457Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:46.639094557Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903}
210042024-02-07T19:03:47.457ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210052024-02-07T19:03:47.462ZLoading write count information from file "/tmp/test_repair_verify.data"
210062024-02-07T19:03:47.462ZRun Repair workload
210072024-02-07T19:03:47.462Z01/30 Write block 359 len 12288 data: 10 13 15
210082024-02-07T19:03:47.462Z02/30 Read block 20 len 28672
210092024-02-07T19:03:47.463Z03/30 Read block 162 len 4096
210102024-02-07T19:03:47.463Z04/30 Read block 492 len 40960
210112024-02-07T19:03:47.463Z05/30 Read block 499 len 8192
210122024-02-07T19:03:47.463Z06/30 Write block 444 len 4096 data: 17
210132024-02-07T19:03:47.463Z07/30 Write block 72 len 8192 data: 11 10
210142024-02-07T19:03:47.463Z08/30 Read block 196 len 4096
210152024-02-07T19:03:47.463Z09/30 Write block 73 len 12288 data: 11 10 11
210162024-02-07T19:03:47.463Z10/30 Write block 426 len 16384 data: 15 13 20 18
210172024-02-07T19:03:47.463Z11/30 Flush
210182024-02-07T19:03:47.464Z12/30 Write block 45 len 28672 data: 8 8 9 10 9 12 15
210192024-02-07T19:03:47.464Z13/30 Read block 158 len 28672
210202024-02-07T19:03:47.464Z14/30 Read block 172 len 20480
210212024-02-07T19:03:47.468Z15/30 Flush
210222024-02-07T19:03:47.468Z16/30 Write block 556 len 36864 data: 17 14 14 16 17 14 15 15 17
210232024-02-07T19:03:47.468Z17/30 Write block 403 len 12288 data: 11 8 10
210242024-02-07T19:03:47.468Z18/30 Flush
210252024-02-07T19:03:47.468Z19/30 Write block 565 len 28672 data: 19 19 20 19 18 19 17
210262024-02-07T19:03:47.468Z20/30 Read block 360 len 20480
210272024-02-07T19:03:47.472Z21/30 Read block 130 len 40960
210282024-02-07T19:03:47.473Z22/30 Flush
210292024-02-07T19:03:47.473Z23/30 Write block 323 len 32768 data: 10 9 10 12 11 11 9 12
210302024-02-07T19:03:47.477Z24/30 Write block 386 len 20480 data: 12 11 14 16 10
210312024-02-07T19:03:47.477Z25/30 Read block 69 len 16384
210322024-02-07T19:03:47.477Z26/30 Read block 86 len 36864
210332024-02-07T19:03:47.477Z27/30 Read block 283 len 28672
210342024-02-07T19:03:47.478Z28/30 Write block 250 len 24576 data: 13 13 14 13 17 14
210352024-02-07T19:03:47.482Z29/30 Write block 181 len 16384 data: 9 8 7 7
210362024-02-07T19:03:47.483Z30/30 Read block 329 len 16384
210372024-02-07T19:03:47.483Z----------------------------------------------------------------
210382024-02-07T19:03:47.483Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
210392024-02-07T19:03:47.483ZStates: Active Active Active
210402024-02-07T19:03:47.483ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
210412024-02-07T19:03:47.483Z 1 Acked 1000 Write 3 Done Sent Done false
210422024-02-07T19:03:47.484Z 2 Acked 1001 Read 7 Done Sent Done false
210432024-02-07T19:03:47.484Z 3 Acked 1002 Read 1 Done Sent Done false
210442024-02-07T19:03:47.484Z 4 Acked 1003 Read 10 Done Sent Done false
210452024-02-07T19:03:47.484Z 5 Acked 1004 Read 2 Done Sent Done false
210462024-02-07T19:03:47.484Z 6 Acked 1005 Write 1 Done Sent Done false
210472024-02-07T19:03:47.484Z 7 Acked 1006 Write 2 Done Sent Done false
210482024-02-07T19:03:47.485Z 8 Acked 1007 Read 1 Done Sent Done false
210492024-02-07T19:03:47.485Z 9 Acked 1008 Write 3 Done Sent Done false
210502024-02-07T19:03:47.485Z 10 Acked 1009 Write 4 Done Sent Done false
210512024-02-07T19:03:47.485Z 11 Acked 1010 Flush 0 Done Sent Done false
210522024-02-07T19:03:47.485Z 12 Acked 1011 Write 7 Done Sent Done false
210532024-02-07T19:03:47.485Z 13 Acked 1012 Read 7 Done Sent Done false
210542024-02-07T19:03:47.485Z 14 Acked 1013 Read 5 Done Sent Done false
210552024-02-07T19:03:47.485Z 15 Acked 1014 Flush 0 Done Sent Done false
210562024-02-07T19:03:47.486Z 16 Acked 1015 Write 9 Done Sent Done false
210572024-02-07T19:03:47.486Z 17 Acked 1016 Write 3 Done Sent Done false
210582024-02-07T19:03:47.486Z 18 Acked 1017 Flush 0 Done Sent Done false
210592024-02-07T19:03:47.486Z 19 Acked 1018 Write 7 Done Sent Done false
210602024-02-07T19:03:47.486Z 20 Acked 1019 Read 5 Done Sent Done false
210612024-02-07T19:03:47.486Z 21 Acked 1020 Read 10 Done Sent Done false
210622024-02-07T19:03:47.486Z 22 Acked 1021 Flush 0 Done Sent Done false
210632024-02-07T19:03:47.486Z 23 Acked 1022 Write 8 Done Sent Done false
210642024-02-07T19:03:47.487Z 24 Acked 1023 Write 5 Done Sent Done false
210652024-02-07T19:03:47.487Z 25 Acked 1024 Read 4 Done Sent Done false
210662024-02-07T19:03:47.487Z 26 Acked 1025 Read 9 Done Sent Done false
210672024-02-07T19:03:47.487Z 27 Acked 1026 Read 7 Done Sent Done false
210682024-02-07T19:03:47.487Z 28 Acked 1027 Write 6 Done Sent Done false
210692024-02-07T19:03:47.487Z 29 Acked 1028 Write 4 Done Sent Done false
210702024-02-07T19:03:47.487Z 30 Acked 1029 Read 4 Done Sent Sent false
210712024-02-07T19:03:47.487Z STATES DS:0 DS:1 DS:2 TOTAL
210722024-02-07T19:03:47.487Z New 0 0 0 0
210732024-02-07T19:03:47.488Z Sent 0 30 1 31
210742024-02-07T19:03:47.488Z Done 30 0 29 59
210752024-02-07T19:03:47.488Z Skipped 0 0 0 0
210762024-02-07T19:03:47.488Z Error 0 0 0 0
210772024-02-07T19:03:47.488ZLast Flush: 1021 0 1021
210782024-02-07T19:03:47.488ZDownstairs last five completed:
210792024-02-07T19:03:47.488ZUpstairs last five completed: 30 29 28 27 26
210802024-02-07T19:03:47.488Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:46.669598982Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1903}
210812024-02-07T19:03:47.488ZWrote out file "/tmp/test_repair_verify.data"
210822024-02-07T19:03:47.493Z
210832024-02-07T19:03:47.493ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
210842024-02-07T19:03:47.503ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210852024-02-07T19:03:47.503Z 0 000-019 190 190 190 334 334 334 F F F
210862024-02-07T19:03:47.503Z 1 020-039 190 190 190 334 334 334 F F F
210872024-02-07T19:03:47.503Z 2 040-059 192 190 192 336 334 336 F F F <---
210882024-02-07T19:03:47.503Z 3 060-079 192 189 192 335 333 335 F F F <---
210892024-02-07T19:03:47.504Z 4 080-099 189 189 189 333 333 333 F F F
210902024-02-07T19:03:47.504Z 5 100-119 188 188 188 332 332 332 F F F
210912024-02-07T19:03:47.508Z 6 120-139 182 182 182 322 322 322 F F F
210922024-02-07T19:03:47.508Z 7 140-159 183 183 183 324 324 324 F F F
210932024-02-07T19:03:47.508Z 8 160-179 186 186 186 328 328 328 F F F
210942024-02-07T19:03:47.508Z 9 180-199 189 189 189 333 333 333 T F T <---
210952024-02-07T19:03:47.508Z 10 200-219 188 188 188 332 332 332 F F F
210962024-02-07T19:03:47.508Z 11 220-239 188 188 188 332 332 332 F F F
210972024-02-07T19:03:47.509Z 12 240-259 186 186 186 329 329 329 T F T <---
210982024-02-07T19:03:47.509Z 13 260-279 190 190 190 334 334 334 F F F
210992024-02-07T19:03:47.509Z 14 280-299 185 185 185 327 327 327 F F F
211002024-02-07T19:03:47.509Z 15 300-319 182 182 182 323 323 323 F F F
211012024-02-07T19:03:47.509Z 16 320-339 190 190 190 334 334 334 T F T <---
211022024-02-07T19:03:47.509Z 17 340-359 192 184 192 335 325 335 F F F <---
211032024-02-07T19:03:47.509Z 18 360-379 192 188 192 335 332 335 F F F <---
211042024-02-07T19:03:47.509Z 19 380-399 188 188 188 332 332 332 T F T <---
211052024-02-07T19:03:47.509Z 20 400-419 192 190 192 337 334 337 F F F <---
211062024-02-07T19:03:47.510Z 21 420-439 192 190 192 335 334 335 F F F <---
211072024-02-07T19:03:47.510Z 22 440-459 192 190 192 335 334 335 F F F <---
211082024-02-07T19:03:47.510Z 23 460-479 190 190 190 334 334 334 F F F
211092024-02-07T19:03:47.510Z 24 480-499 187 187 187 330 330 330 F F F
211102024-02-07T19:03:47.510Z 25 500-519 186 186 186 328 328 328 F F F
211112024-02-07T19:03:47.510Z 26 520-539 187 187 187 330 330 330 F F F
211122024-02-07T19:03:47.510Z 27 540-559 192 187 192 337 330 337 F F F <---
211132024-02-07T19:03:47.511Z 28 560-579 192 190 192 338 334 338 F F F <---
211142024-02-07T19:03:47.511Z 29 580-599 190 190 190 334 334 334 F F F
211152024-02-07T19:03:47.511ZMax gen: 192, Max flush: 338
211162024-02-07T19:03:47.511ZError: Difference in extent metadata found!
211172024-02-07T19:03:47.511ZA Difference in extent metadata is expected here
211182024-02-07T19:03:47.511ZOn loop 96
211192024-02-07T19:03:47.511Z
211202024-02-07T19:03:47.516ZVerifying data now
211212024-02-07T19:03:52.161ZNo change after verify
211222024-02-07T19:03:52.161ZLoop: 96 Downstairs dump after verify (and repair):
211232024-02-07T19:03:52.166ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211242024-02-07T19:03:52.166Z 0 000-019 190 190 190 334 334 334 F F F
211252024-02-07T19:03:52.166Z 1 020-039 190 190 190 334 334 334 F F F
211262024-02-07T19:03:52.166Z 2 040-059 192 192 192 336 336 336 F F F
211272024-02-07T19:03:52.166Z 3 060-079 192 192 192 335 335 335 F F F
211282024-02-07T19:03:52.166Z 4 080-099 189 189 189 333 333 333 F F F
211292024-02-07T19:03:52.167Z 5 100-119 188 188 188 332 332 332 F F F
211302024-02-07T19:03:52.167Z 6 120-139 182 182 182 322 322 322 F F F
211312024-02-07T19:03:52.167Z 7 140-159 183 183 183 324 324 324 F F F
211322024-02-07T19:03:52.167Z 8 160-179 186 186 186 328 328 328 F F F
211332024-02-07T19:03:52.167Z 9 180-199 193 193 193 339 339 339 F F F
211342024-02-07T19:03:52.167Z 10 200-219 188 188 188 332 332 332 F F F
211352024-02-07T19:03:52.167Z 11 220-239 188 188 188 332 332 332 F F F
211362024-02-07T19:03:52.167Z 12 240-259 193 193 193 339 339 339 F F F
211372024-02-07T19:03:52.167Z 13 260-279 190 190 190 334 334 334 F F F
211382024-02-07T19:03:52.167Z 14 280-299 185 185 185 327 327 327 F F F
211392024-02-07T19:03:52.167Z 15 300-319 182 182 182 323 323 323 F F F
211402024-02-07T19:03:52.168Z 16 320-339 193 193 193 339 339 339 F F F
211412024-02-07T19:03:52.168Z 17 340-359 192 192 192 335 335 335 F F F
211422024-02-07T19:03:52.168Z 18 360-379 192 192 192 335 335 335 F F F
211432024-02-07T19:03:52.168Z 19 380-399 193 193 193 339 339 339 F F F
211442024-02-07T19:03:52.168Z 20 400-419 192 192 192 337 337 337 F F F
211452024-02-07T19:03:52.168Z 21 420-439 192 192 192 335 335 335 F F F
211462024-02-07T19:03:52.168Z 22 440-459 192 192 192 335 335 335 F F F
211472024-02-07T19:03:52.168Z 23 460-479 190 190 190 334 334 334 F F F
211482024-02-07T19:03:52.168Z 24 480-499 187 187 187 330 330 330 F F F
211492024-02-07T19:03:52.169Z 25 500-519 186 186 186 328 328 328 F F F
211502024-02-07T19:03:52.169Z 26 520-539 187 187 187 330 330 330 F F F
211512024-02-07T19:03:52.169Z 27 540-559 192 192 192 337 337 337 F F F
211522024-02-07T19:03:52.169Z 28 560-579 192 192 192 338 338 338 F F F
211532024-02-07T19:03:52.169Z 29 580-599 190 190 190 334 334 334 F F F
211542024-02-07T19:03:52.169ZMax gen: 193, Max flush: 339
211552024-02-07T19:03:52.174Z
211562024-02-07T19:03:52.174ZBegin loop 97
211572024-02-07T19:03:52.174ZDownstairs to restart: 2
211582024-02-07T19:03:52.187Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.376451959Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911}
211592024-02-07T19:03:52.194Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
211602024-02-07T19:03:52.194Z,"time":"2024-02-07T19:03:51.37681151Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911}
211612024-02-07T19:03:52.194Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.376860634Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911}
211622024-02-07T19:03:52.194Z{"msg":"Crucible 9d1ad0b6-a51d-4c56-af5d-2d6b2177648e has session id: 38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.376899079Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211632024-02-07T19:03:52.194Z{"msg":"Upstairs opts: Upstairs UUID: 9d1ad0b6-a51d-4c56-af5d-2d6b2177648e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.376936323Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211642024-02-07T19:03:52.194Z{"msg":"Crucible stats registered with UUID: 9d1ad0b6-a51d-4c56-af5d-2d6b2177648e","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.376955461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211652024-02-07T19:03:52.194Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.376971509Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911}
211662024-02-07T19:03:52.195Z{"msg":"9d1ad0b6-a51d-4c56-af5d-2d6b2177648e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377137936Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211672024-02-07T19:03:52.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377173991Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211682024-02-07T19:03:52.195Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377218835Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211692024-02-07T19:03:52.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.37725314Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211702024-02-07T19:03:52.195Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377287456Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211712024-02-07T19:03:52.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377336889Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211722024-02-07T19:03:52.195Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377385153Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211732024-02-07T19:03:52.195Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377433876Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211742024-02-07T19:03:52.195Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377471771Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211752024-02-07T19:03:52.196Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377497047Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211762024-02-07T19:03:52.196Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:51.37755116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211772024-02-07T19:03:52.196Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:51.377584406Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211782024-02-07T19:03:52.196Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377610582Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211792024-02-07T19:03:52.196Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:51.377639968Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211802024-02-07T19:03:52.196Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377668954Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211812024-02-07T19:03:52.196Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.37770101Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211822024-02-07T19:03:52.196Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377730656Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211832024-02-07T19:03:52.196Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.377754863Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211842024-02-07T19:03:52.196Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.378152209Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211852024-02-07T19:03:52.197Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.378200312Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211862024-02-07T19:03:52.197Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.378261574Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211872024-02-07T19:03:52.197Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.378305858Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211882024-02-07T19:03:52.197Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.378844095Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211892024-02-07T19:03:52.197Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.378897888Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211902024-02-07T19:03:52.197Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.379514395Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211912024-02-07T19:03:52.197Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.379567327Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211922024-02-07T19:03:52.198Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.379612632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211932024-02-07T19:03:52.198Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.379648217Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211942024-02-07T19:03:52.198Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.379665694Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211952024-02-07T19:03:52.198Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:51.379687131Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211962024-02-07T19:03:53.189Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.378532187Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211972024-02-07T19:03:53.210Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.378853213Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211982024-02-07T19:03:53.210Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.379832461Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
211992024-02-07T19:03:53.210Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.379868086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212002024-02-07T19:03:53.210Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.380416632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212012024-02-07T19:03:53.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381032978Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212022024-02-07T19:03:53.210Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381075983Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212032024-02-07T19:03:53.210Z{"msg":"[0]R flush_numbers[0..12]: [334, 334, 336, 335, 333, 332, 322, 324, 328, 339, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.38109579Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212042024-02-07T19:03:53.210Z{"msg":"[0]R generation[0..12]: [190, 190, 192, 192, 189, 188, 182, 183, 186, 193, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381123376Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212052024-02-07T19:03:53.210Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381142534Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212062024-02-07T19:03:53.211Z{"msg":"[1]R flush_numbers[0..12]: [334, 334, 336, 335, 333, 332, 322, 324, 328, 339, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.38116915Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212072024-02-07T19:03:53.211Z{"msg":"[1]R generation[0..12]: [190, 190, 192, 192, 189, 188, 182, 183, 186, 193, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381187488Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212082024-02-07T19:03:53.211Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381213664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212092024-02-07T19:03:53.211Z{"msg":"[2]R flush_numbers[0..12]: [334, 334, 336, 335, 333, 332, 322, 324, 328, 339, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381233542Z"Wait for a query_work_queue command to finish before sending IO
212102024-02-07T19:03:53.211Z,"hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212112024-02-07T19:03:53.211Z{"msg":"[2]R generation[0..12]: [190, 190, 192, 192, 189, 188, 182, 183, 186, 193, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381264317Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212122024-02-07T19:03:53.211Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381282755Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212132024-02-07T19:03:53.211Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381307951Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212142024-02-07T19:03:53.211Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381325629Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212152024-02-07T19:03:53.211Z{"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381350196Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212162024-02-07T19:03:53.211Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381367643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212172024-02-07T19:03:53.211Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381399719Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212182024-02-07T19:03:53.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381415797Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212192024-02-07T19:03:53.212Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.38146443Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212202024-02-07T19:03:53.212Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381494786Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212212024-02-07T19:03:53.212Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381532061Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212222024-02-07T19:03:53.212Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212232024-02-07T19:03:53.212ZSet Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381563007Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212242024-02-07T19:03:53.212Z{"msg":"9d1ad0b6-a51d-4c56-af5d-2d6b2177648e is now active with session: 38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381599552Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212252024-02-07T19:03:53.212Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381627548Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911,"session_id":"38f41a78-c3c1-44d8-9fcd-7ffe27fd80e9"}
212262024-02-07T19:03:53.212Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:52.381660304Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911}
212272024-02-07T19:03:53.212ZLoading write count information from file "/tmp/test_repair_verify.data"
212282024-02-07T19:03:53.212ZRun Repair workload
212292024-02-07T19:03:53.212Z01/30 Write block 184 len 16384 data: 8 12 11 13
212302024-02-07T19:03:53.217Z02/30 Read block 294 len 32768
212312024-02-07T19:03:53.217Z03/30 Read block 390 len 32768
212322024-02-07T19:03:53.217Z04/30 Write block 261 len 4096 data: 12
212332024-02-07T19:03:53.217Z05/30 Write block 213 len 4096 data: 13
212342024-02-07T19:03:53.218Z06/30 Read block 478 len 20480
212352024-02-07T19:03:53.218Z07/30 Flush
212362024-02-07T19:03:53.218Z08/30 Write block 453 len 8192 data: 14 14
212372024-02-07T19:03:53.218Z09/30 Read block 291 len 16384
212382024-02-07T19:03:53.218Z10/30 Read block 228 len 12288
212392024-02-07T19:03:53.218Z11/30 Write block 111 len 8192 data: 16 14
212402024-02-07T19:03:53.218Z12/30 Read block 64 len 4096
212412024-02-07T19:03:53.218Z13/30 Write block 53 len 4096 data: 13
212422024-02-07T19:03:53.218Z14/30 Read block 203 len 28672
212432024-02-07T19:03:53.218Z15/30 Read block 71 len 40960
212442024-02-07T19:03:53.218Z16/30 Read block 367 len 8192
212452024-02-07T19:03:53.218Z17/30 Read block 360 len 40960
212462024-02-07T19:03:53.218Z18/30 Read block 414 len 36864
212472024-02-07T19:03:53.219Z19/30 Read block 276 len 8192
212482024-02-07T19:03:53.219Z20/30 Write block 562 len 4096 data: 16
212492024-02-07T19:03:53.219Z21/30 Read block 503 len 28672
212502024-02-07T19:03:53.219Z22/30 Write block 503 len 28672 data: 17 15 18 16 14 15 15
212512024-02-07T19:03:53.219Z23/30 Write block 545 len 28672 data: 11 13 18 19 20 19 16
212522024-02-07T19:03:53.219Z24/30 Flush
212532024-02-07T19:03:53.219Z25/30 Write block 406 len 16384 data: 11 10 10 12
212542024-02-07T19:03:53.219Z26/30 Read block 257 len 40960
212552024-02-07T19:03:53.224Z27/30 Read block 89 len 32768
212562024-02-07T19:03:53.224Z28/30 Read block 492 len 28672
212572024-02-07T19:03:53.224Z29/30 Read block 361 len 8192
212582024-02-07T19:03:53.224Z30/30 Read block 129 len 40960
212592024-02-07T19:03:53.225Z----------------------------------------------------------------
212602024-02-07T19:03:53.225Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
212612024-02-07T19:03:53.225ZStates: Active Active Active
212622024-02-07T19:03:53.225ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
212632024-02-07T19:03:53.225Z 1 Acked 1000 Write 4 Done Done Sent false
212642024-02-07T19:03:53.225Z 2 Acked 1001 Read 8 Done Done Sent false
212652024-02-07T19:03:53.225Z 3 Acked 1002 Read 8 Done Done Sent false
212662024-02-07T19:03:53.225Z 4 Acked 1003 Write 1 Done Done Sent false
212672024-02-07T19:03:53.225Z 5 Acked 1004 Write 1 Done Done Sent false
212682024-02-07T19:03:53.225Z 6 Acked 1005 Read 5 Done Done Sent false
212692024-02-07T19:03:53.225Z 7 Acked 1006 Flush 0 Done Done Sent false
212702024-02-07T19:03:53.225Z 8 Acked 1007 Write 2 Done Done Sent false
212712024-02-07T19:03:53.225Z 9 Acked 1008 Read 4 Done Done Sent false
212722024-02-07T19:03:53.226Z 10 Acked 1009 Read 3 Done Done Sent false
212732024-02-07T19:03:53.226Z 11 Acked 1010 Write 2 Done Done Sent false
212742024-02-07T19:03:53.226Z 12 Acked 1011 Read 1 Done Done Sent false
212752024-02-07T19:03:53.226Z 13 Acked 1012 Write 1 Done Done Sent false
212762024-02-07T19:03:53.226Z 14 Acked 1013 Read 7 Done Done Sent false
212772024-02-07T19:03:53.226Z 15 Acked 1014 Read 10 Done Done Sent false
212782024-02-07T19:03:53.226Z 16 Acked 1015 Read 2 Done Done Sent false
212792024-02-07T19:03:53.226Z 17 Acked 1016 Read 10 Done Done Sent false
212802024-02-07T19:03:53.226Z 18 Acked 1017 Read 9 Done Done Sent false
212812024-02-07T19:03:53.226Z 19 Acked 1018 Read 2 Done Done Sent false
212822024-02-07T19:03:53.226Z 20 Acked 1019 Write 1 Done Done Sent false
212832024-02-07T19:03:53.226Z 21 Acked 1020 Read 7 Done Done Sent false
212842024-02-07T19:03:53.226Z 22 Acked 1021 Write 7 Done Done Sent false
212852024-02-07T19:03:53.226Z 23 Acked 1022 Write 7 Done Done Sent false
212862024-02-07T19:03:53.226Z 24 Acked 1023 Flush 0 Done Done Sent false
212872024-02-07T19:03:53.226Z 25 Acked 1024 Write 4 Done Done Sent false
212882024-02-07T19:03:53.226Z 26 Acked 1025 Read 10 Done Done Sent false
212892024-02-07T19:03:53.226Z 27 Acked 1026 Read 8 Done Done Sent false
212902024-02-07T19:03:53.226Z 28 Acked 1027 Read 7 Done Done Sent false
212912024-02-07T19:03:53.226Z 29 Acked 1028 Read 2 Done Done Sent false
212922024-02-07T19:03:53.227Z 30 Acked 1029 Read 10 Done Done Sent false
212932024-02-07T19:03:53.227Z STATES DS:0 DS:1 DS:2 TOTAL
212942024-02-07T19:03:53.227Z New 0 0 0 0
212952024-02-07T19:03:53.227Z Sent 0 0 30 30
212962024-02-07T19:03:53.227Z Done 30 30 0 60
212972024-02-07T19:03:53.227Z Skipped 0 0 0 0
212982024-02-07T19:03:53.227Z Error 0 0 0 0
212992024-02-07T19:03:53.227ZLast Flush: 1023 1023 0
213002024-02-07T19:03:53.227ZDownstairs last five completed:
213012024-02-07T19:03:53.227ZUpstairs last five completed: 30 29 28 27 26
213022024-02-07T19:03:53.227Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:52.410472907Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1911}
213032024-02-07T19:03:53.227ZWrote out file "/tmp/test_repair_verify.data"
213042024-02-07T19:03:53.232Z
213052024-02-07T19:03:53.232ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213062024-02-07T19:03:53.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213072024-02-07T19:03:53.244Z 0 000-019 190 190 190 334 334 334 F F F
213082024-02-07T19:03:53.244Z 1 020-039 190 190 190 334 334 334 F F F
213092024-02-07T19:03:53.245Z 2 040-059 194 194 192 341 341 336 F F F <---
213102024-02-07T19:03:53.245Z 3 060-079 192 192 192 335 335 335 F F F
213112024-02-07T19:03:53.245Z 4 080-099 189 189 189 333 333 333 F F F
213122024-02-07T19:03:53.245Z 5 100-119 194 194 188 341 341 332 F F F <---
213132024-02-07T19:03:53.245Z 6 120-139 182 182 182 322 322 322 F F F
213142024-02-07T19:03:53.245Z 7 140-159 183 183 183 324 324 324 F F F
213152024-02-07T19:03:53.245Z 8 160-179 186 186 186 328 328 328 F F F
213162024-02-07T19:03:53.245Z 9 180-199 194 194 193 340 340 339 F F F <---
213172024-02-07T19:03:53.261Z 10 200-219 194 194 188 340 340 332 F F F <---
213182024-02-07T19:03:53.261Z 11 220-239 188 188 188 332 332 332 F F F
213192024-02-07T19:03:53.261Z 12 240-259 193 193 193 339 339 339 F F F
213202024-02-07T19:03:53.261Z 13 260-279 194 194 190 340 340 334 F F F <---
213212024-02-07T19:03:53.261Z 14 280-299 185 185 185 327 327 327 F F F
213222024-02-07T19:03:53.261Z 15 300-319 182 182 182 323 323 323 F F F
213232024-02-07T19:03:53.262Z 16 320-339 193 193 193 339 339 339 F F F
213242024-02-07T19:03:53.262Z 17 340-359 192 192 192 335 335 335 F F F
213252024-02-07T19:03:53.262Z 18 360-379 192 192 192 335 335 335 F F F
213262024-02-07T19:03:53.262Z 19 380-399 193 193 193 339 339 339 F F F
213272024-02-07T19:03:53.262Z 20 400-419 192 192 192 337 337 337 T T F <---
213282024-02-07T19:03:53.262Z 21 420-439 192 192 192 335 335 335 F F F
213292024-02-07T19:03:53.262Z 22 440-459 194 194 192 341 341 335 F F F <---
213302024-02-07T19:03:53.262Z 23 460-479 190 190 190 334 334 334 F F F
213312024-02-07T19:03:53.262Z 24 480-499 187 187 187 330 330 330 F F F
213322024-02-07T19:03:53.262Z 25 500-519 194 194 186 341 341 328 F F F <---
213332024-02-07T19:03:53.262Z 26 520-539 187 187 187 330 330 330 F F F
213342024-02-07T19:03:53.262Z 27 540-559 194 194 192 341 341 337 F F F <---
213352024-02-07T19:03:53.262Z 28 560-579 194 194 192 341 341 338 F F F <---
213362024-02-07T19:03:53.263Z 29 580-599 190 190 190 334 334 334 F F F
213372024-02-07T19:03:53.263ZMax gen: 194, Max flush: 341
213382024-02-07T19:03:53.263ZError: Difference in extent metadata found!
213392024-02-07T19:03:53.263ZA Difference in extent metadata is expected here
213402024-02-07T19:03:53.263ZOn loop 97
213412024-02-07T19:03:53.263Z
213422024-02-07T19:03:53.267ZVerifying data now
213432024-02-07T19:03:57.751ZNo change after verify
213442024-02-07T19:03:57.751ZLoop: 97 Downstairs dump after verify (and repair):
213452024-02-07T19:03:57.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213462024-02-07T19:03:57.767Z 0 000-019 190 190 190 334 334 334 F F F
213472024-02-07T19:03:57.767Z 1 020-039 190 190 190 334 334 334 F F F
213482024-02-07T19:03:57.767Z 2 040-059 194 194 194 341 341 341 F F F
213492024-02-07T19:03:57.767Z 3 060-079 192 192 192 335 335 335 F F F
213502024-02-07T19:03:57.767Z 4 080-099 189 189 189 333 333 333 F F F
213512024-02-07T19:03:57.767Z 5 100-119 194 194 194 341 341 341 F F F
213522024-02-07T19:03:57.767Z 6 120-139 182 182 182 322 322 322 F F F
213532024-02-07T19:03:57.767Z 7 140-159 183 183 183 324 324 324 F F F
213542024-02-07T19:03:57.767Z 8 160-179 186 186 186 328 328 328 F F F
213552024-02-07T19:03:57.767Z 9 180-199 194 194 194 340 340 340 F F F
213562024-02-07T19:03:57.767Z 10 200-219 194 194 194 340 340 340 F F F
213572024-02-07T19:03:57.767Z 11 220-239 188 188 188 332 332 332 F F F
213582024-02-07T19:03:57.767Z 12 240-259 193 193 193 339 339 339 F F F
213592024-02-07T19:03:57.767Z 13 260-279 194 194 194 340 340 340 F F F
213602024-02-07T19:03:57.767Z 14 280-299 185 185 185 327 327 327 F F F
213612024-02-07T19:03:57.771Z 15 300-319 182 182 182 323 323 323 F F F
213622024-02-07T19:03:57.771Z 16 320-339 193 193 193 339 339 339 F F F
213632024-02-07T19:03:57.771Z 17 340-359 192 192 192 335 335 335 F F F
213642024-02-07T19:03:57.771Z 18 360-379 192 192 192 335 335 335 F F F
213652024-02-07T19:03:57.771Z 19 380-399 193 193 193 339 339 339 F F F
213662024-02-07T19:03:57.771Z 20 400-419 195 195 195 342 342 342 F F F
213672024-02-07T19:03:57.771Z 21 420-439 192 192 192 335 335 335 F F F
213682024-02-07T19:03:57.771Z 22 440-459 194 194 194 341 341 341 F F F
213692024-02-07T19:03:57.771Z 23 460-479 190 190 190 334 334 334 F F F
213702024-02-07T19:03:57.771Z 24 480-499 187 187 187 330 330 330 F F F
213712024-02-07T19:03:57.771Z 25 500-519 194 194 194 341 341 341 F F F
213722024-02-07T19:03:57.771Z 26 520-539 187 187 187 330 330 330 F F F
213732024-02-07T19:03:57.771Z 27 540-559 194 194 194 341 341 341 F F F
213742024-02-07T19:03:57.771Z 28 560-579 194 194 194 341 341 341 F F F
213752024-02-07T19:03:57.771Z 29 580-599 190 190 190 334 334 334 F F F
213762024-02-07T19:03:57.771ZMax gen: 195, Max flush: 342
213772024-02-07T19:03:57.771Z
213782024-02-07T19:03:57.771ZBegin loop 98
213792024-02-07T19:03:57.771ZDownstairs to restart: 0
213802024-02-07T19:03:57.788Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977545048Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919}
213812024-02-07T19:03:57.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977819091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919}
213822024-02-07T19:03:57.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977837898Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919}
213832024-02-07T19:03:57.793Z{"msg":"Crucible 30a19124-0672-4888-a78d-cd5c25183386 has session id: 8d746c9c-46b4-4967-acbb-23600ec4e17a","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977870174Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919Crucible runtime is spawned
213842024-02-07T19:03:57.794Z,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213852024-02-07T19:03:57.794Z{"msg":"Upstairs opts: Upstairs UUID: 30a19124-0672-4888-a78d-cd5c25183386, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977914578Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213862024-02-07T19:03:57.794Z{"msg":"Crucible stats registered with UUID: 30a19124-0672-4888-a78d-cd5c25183386","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977941395Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213872024-02-07T19:03:57.794Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.977962662Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919}
213882024-02-07T19:03:57.794Z{"msg":"30a19124-0672-4888-a78d-cd5c25183386 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978133269Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213892024-02-07T19:03:57.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978191241Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213902024-02-07T19:03:57.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978231975Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213912024-02-07T19:03:57.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978259972Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213922024-02-07T19:03:57.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978285918Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213932024-02-07T19:03:57.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978323533Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213942024-02-07T19:03:57.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978349679Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213952024-02-07T19:03:57.794Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978392343Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213962024-02-07T19:03:57.794Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.97842088Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213972024-02-07T19:03:57.794Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978457765Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213982024-02-07T19:03:57.794Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.97849419Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
213992024-02-07T19:03:57.794Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978655098Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214002024-02-07T19:03:57.794Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.978702661Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214012024-02-07T19:03:57.794Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.979090539Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214022024-02-07T19:03:57.794Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.97915466Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214032024-02-07T19:03:57.794Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.97922737Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214042024-02-07T19:03:57.794Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.979273964Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214052024-02-07T19:03:57.794Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.979664302Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214062024-02-07T19:03:57.794Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.979835828Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214072024-02-07T19:03:57.795Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980237804Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214082024-02-07T19:03:57.795Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980275549Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214092024-02-07T19:03:57.795Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980309844Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214102024-02-07T19:03:57.795Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.98033859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214112024-02-07T19:03:57.795Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980353878Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214122024-02-07T19:03:57.795Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980476012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214132024-02-07T19:03:57.795Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980503478Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214142024-02-07T19:03:57.795Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980518316Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214152024-02-07T19:03:57.795Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.980937219Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214162024-02-07T19:03:57.800Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981391958Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214172024-02-07T19:03:57.800Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981428043Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214182024-02-07T19:03:57.800Z{"msg":"[0]R flush_numbers[0..12]: [334, 334, 341, 335, 333, 341, 322, 324, 328, 340, 340, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981444401Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214192024-02-07T19:03:57.800Z{"msg":"[0]R generation[0..12]: [190, 190, 194, 192, 189, 194, 182, 183, 186, 194, 194, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981466078Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214202024-02-07T19:03:57.800Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981481046Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214212024-02-07T19:03:57.800Z{"msg":"[1]R flush_numbers[0..12]: [334, 334, 341, 335, 333, 341, 322, 324, 328, 340, 340, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981502223Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214222024-02-07T19:03:57.800Z{"msg":"[1]R generation[0..12]: [190, 190, 194, 192, 189, 194, 182, 183, 186, 194, 194, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981516851Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214232024-02-07T19:03:57.800Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981537518Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214242024-02-07T19:03:57.800Z{"msg":"[2]R flush_numbers[0..12]: [334, 334, 341, 335, 333, 341, 322, 324, 328, 340, 340, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981552216Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214252024-02-07T19:03:57.800Z{"msg":"[2]R generation[0..12]: [190, 190, 194, 192, 189, 194, 182, 183, 186, 194, 194, 188]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO
214262024-02-07T19:03:57.800Z","level":30,"time":"2024-02-07T19:03:56.981575503Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214272024-02-07T19:03:57.800Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981590621Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214282024-02-07T19:03:57.800Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981616438Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214292024-02-07T19:03:57.800Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981630436Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214302024-02-07T19:03:57.800Z{"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981643404Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214312024-02-07T19:03:57.800Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.98167442Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214322024-02-07T19:03:57.800Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981702316Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214332024-02-07T19:03:57.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981736051Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214342024-02-07T19:03:57.801Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981775796Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214352024-02-07T19:03:57.801Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981812901Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214362024-02-07T19:03:57.801Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981850116Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214372024-02-07T19:03:57.801Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981886181Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214382024-02-07T19:03:57.801Z{"msg":"30a19124-0672-4888-a78d-cd5c25183386 is now active with session: 8d746c9c-46b4-4967-acbb-23600ec4e17a","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981922786Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214392024-02-07T19:03:57.801Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981957671Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919,"session_id":"8d746c9c-46b4-4967-acbb-23600ec4e17a"}
214402024-02-07T19:03:57.801Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:56.981992716Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919}
214412024-02-07T19:03:57.801ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
214422024-02-07T19:03:57.801ZLoading write count information from file "/tmp/test_repair_verify.data"
214432024-02-07T19:03:57.801ZRun Repair workload
214442024-02-07T19:03:57.801Z01/30 Write block 263 len 40960 data: 18 21 17 17 18 14 13 10 10 11
214452024-02-07T19:03:57.801Z02/30 Read block 583 len 20480
214462024-02-07T19:03:57.801Z03/30 Write block 276 len 24576 data: 11 11 12 12 13 15
214472024-02-07T19:03:57.801Z04/30 Write block 126 len 8192 data: 21 20
214482024-02-07T19:03:57.801Z05/30 Read block 544 len 28672
214492024-02-07T19:03:57.805Z06/30 Read block 248 len 36864
214502024-02-07T19:03:57.805Z07/30 Read block 135 len 28672
214512024-02-07T19:03:57.805Z08/30 Write block 506 len 40960 data: 17 15 16 16 17 15 19 17 20 21
214522024-02-07T19:03:57.805Z09/30 Read block 366 len 36864
214532024-02-07T19:03:57.805Z10/30 Write block 214 len 20480 data: 10 8 8 8 10
214542024-02-07T19:03:57.805Z11/30 Read block 479 len 8192
214552024-02-07T19:03:57.805Z12/30 Read block 313 len 28672
214562024-02-07T19:03:57.808Z13/30 Read block 404 len 4096
214572024-02-07T19:03:57.808Z14/30 Write block 273 len 20480 data: 12 10 12 12 12
214582024-02-07T19:03:57.808Z15/30 Write block 58 len 4096 data: 13
214592024-02-07T19:03:57.808Z16/30 Read block 104 len 36864
214602024-02-07T19:03:57.811Z17/30 Read block 540 len 4096
214612024-02-07T19:03:57.811Z18/30 Flush
214622024-02-07T19:03:57.814Z19/30 Write block 357 len 28672 data: 6 8 11 14 16 17 17
214632024-02-07T19:03:57.814Z20/30 Read block 130 len 28672
214642024-02-07T19:03:57.817Z21/30 Write block 205 len 24576 data: 15 12 12 14 11 10
214652024-02-07T19:03:57.817Z22/30 Read block 252 len 40960
214662024-02-07T19:03:57.817Z23/30 Read block 18 len 20480
214672024-02-07T19:03:57.818Z24/30 Read block 416 len 40960
214682024-02-07T19:03:57.821Z25/30 Write block 405 len 32768 data: 11 12 11 11 13 15 17 15
214692024-02-07T19:03:57.821Z26/30 Write block 422 len 16384 data: 11 12 13 18
214702024-02-07T19:03:57.821Z27/30 Write block 282 len 36864 data: 14 12 18 18 19 22 20 20 22
214712024-02-07T19:03:57.821Z28/30 Read block 229 len 36864
214722024-02-07T19:03:57.824Z29/30 Write block 210 len 8192 data: 11 12
214732024-02-07T19:03:57.824Z30/30 Write block 274 len 20480 data: 11 13 13 13 13
214742024-02-07T19:03:57.824Z----------------------------------------------------------------
214752024-02-07T19:03:57.824Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
214762024-02-07T19:03:57.824ZStates: Active Active Active
214772024-02-07T19:03:57.824ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
214782024-02-07T19:03:57.824Z 1 Acked 1000 Write 10 Sent Done Done false
214792024-02-07T19:03:57.824Z 2 Acked 1001 Read 5 Sent Done Done false
214802024-02-07T19:03:57.824Z 3 Acked 1002 Write 6 Sent Done Done false
214812024-02-07T19:03:57.825Z 4 Acked 1003 Write 2 Sent Done Done false
214822024-02-07T19:03:57.825Z 5 Acked 1004 Read 7 Sent Done Done false
214832024-02-07T19:03:57.825Z 6 Acked 1005 Read 9 Sent Done Done false
214842024-02-07T19:03:57.825Z 7 Acked 1006 Read 7 Sent Done Done false
214852024-02-07T19:03:57.825Z 8 Acked 1007 Write 10 Sent Done Done false
214862024-02-07T19:03:57.825Z 9 Acked 1008 Read 9 Sent Done Done false
214872024-02-07T19:03:57.825Z 10 Acked 1009 Write 5 Sent Done Done false
214882024-02-07T19:03:57.825Z 11 Acked 1010 Read 2 Sent Done Done false
214892024-02-07T19:03:57.825Z 12 Acked 1011 Read 7 Sent Done Done false
214902024-02-07T19:03:57.825Z 13 Acked 1012 Read 1 Sent Done Done false
214912024-02-07T19:03:57.825Z 14 Acked 1013 Write 5 Sent Done Done false
214922024-02-07T19:03:57.825Z 15 Acked 1014 Write 1 Sent Done Done false
214932024-02-07T19:03:57.825Z 16 Acked 1015 Read 9 Sent Done Done false
214942024-02-07T19:03:57.825Z 17 Acked 1016 Read 1 Sent Done Done false
214952024-02-07T19:03:57.825Z 18 Acked 1017 Flush 0 Sent Done Done false
214962024-02-07T19:03:57.825Z 19 Acked 1018 Write 7 Sent Done Done false
214972024-02-07T19:03:57.825Z 20 Acked 1019 Read 7 Sent Done Done false
214982024-02-07T19:03:57.825Z 21 Acked 1020 Write 6 Sent Done Done false
214992024-02-07T19:03:57.825Z 22 Acked 1021 Read 10 Sent Done Done false
215002024-02-07T19:03:57.825Z 23 Acked 1022 Read 5 Sent Done Done false
215012024-02-07T19:03:57.825Z 24 Acked 1023 Read 10 Sent Done Done false
215022024-02-07T19:03:57.825Z 25 Acked 1024 Write 8 Sent Done Done false
215032024-02-07T19:03:57.825Z 26 Acked 1025 Write 4 Sent Done Done false
215042024-02-07T19:03:57.825Z 27 Acked 1026 Write 9 Sent Done Done false
215052024-02-07T19:03:57.826Z 28 Acked 1027 Read 9 Sent Done Done false
215062024-02-07T19:03:57.826Z 29 Acked 1028 Write 2 Sent Done Done false
215072024-02-07T19:03:57.826Z 30 Acked 1029 Write 5 Sent Sent Sent false
215082024-02-07T19:03:57.826Z STATES DS:0 DS:1 DS:2 TOTAL
215092024-02-07T19:03:57.826Z New 0 0 0 0
215102024-02-07T19:03:57.826Z Sent 30 1 1 32
215112024-02-07T19:03:57.826Z Done 0 29 29 58
215122024-02-07T19:03:57.826Z Skipped 0 0 0 0
215132024-02-07T19:03:57.826Z Error 0 0 0 0
215142024-02-07T19:03:57.826ZLast Flush: 0 1017 1017
215152024-02-07T19:03:57.826ZDownstairs last five completed:
215162024-02-07T19:03:57.826ZUpstairs last five completed: 30 29 28 27 26
215172024-02-07T19:03:57.826Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:57.013322859Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1919}
215182024-02-07T19:03:57.829ZWrote out file "/tmp/test_repair_verify.data"
215192024-02-07T19:03:57.830Z
215202024-02-07T19:03:57.832ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
215212024-02-07T19:03:57.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215222024-02-07T19:03:57.851Z 0 000-019 190 190 190 334 334 334 F F F
215232024-02-07T19:03:57.851Z 1 020-039 190 190 190 334 334 334 F F F
215242024-02-07T19:03:57.851Z 2 040-059 194 196 196 341 343 343 F F F <---
215252024-02-07T19:03:57.851Z 3 060-079 192 192 192 335 335 335 F F F
215262024-02-07T19:03:57.851Z 4 080-099 189 189 189 333 333 333 F F F
215272024-02-07T19:03:57.851Z 5 100-119 194 194 194 341 341 341 F F F
215282024-02-07T19:03:57.851Z 6 120-139 182 196 196 322 343 343 F F F <---
215292024-02-07T19:03:57.851Z 7 140-159 183 183 183 324 324 324 F F F
215302024-02-07T19:03:57.851Z 8 160-179 186 186 186 328 328 328 F F F
215312024-02-07T19:03:57.851Z 9 180-199 194 194 194 340 340 340 F F F
215322024-02-07T19:03:57.851Z 10 200-219 194 196 196 340 343 343 F T T <---
215332024-02-07T19:03:57.851Z 11 220-239 188 188 188 332 332 332 F F F
215342024-02-07T19:03:57.851Z 12 240-259 193 193 193 339 339 339 F F F
215352024-02-07T19:03:57.851Z 13 260-279 194 196 196 340 343 343 F T T <---
215362024-02-07T19:03:57.858Z 14 280-299 185 196 196 327 343 343 F T T <---
215372024-02-07T19:03:57.858Z 15 300-319 182 182 182 323 323 323 F F F
215382024-02-07T19:03:57.858Z 16 320-339 193 193 193 339 339 339 F F F
215392024-02-07T19:03:57.858Z 17 340-359 192 192 192 335 335 335 F T T <---
215402024-02-07T19:03:57.858Z 18 360-379 192 192 192 335 335 335 F T T <---
215412024-02-07T19:03:57.858Z 19 380-399 193 193 193 339 339 339 F F F
215422024-02-07T19:03:57.858Z 20 400-419 195 195 195 342 342 342 F T T <---
215432024-02-07T19:03:57.858Z 21 420-439 192 192 192 335 335 335 F T T <---
215442024-02-07T19:03:57.858Z 22 440-459 194 194 194 341 341 341 F F F
215452024-02-07T19:03:57.858Z 23 460-479 190 190 190 334 334 334 F F F
215462024-02-07T19:03:57.858Z 24 480-499 187 187 187 330 330 330 F F F
215472024-02-07T19:03:57.858Z 25 500-519 194 196 196 341 343 343 F F F <---
215482024-02-07T19:03:57.858Z 26 520-539 187 187 187 330 330 330 F F F
215492024-02-07T19:03:57.858Z 27 540-559 194 194 194 341 341 341 F F F
215502024-02-07T19:03:57.858Z 28 560-579 194 194 194 341 341 341 F F F
215512024-02-07T19:03:57.858Z 29 580-599 190 190 190 334 334 334 F F F
215522024-02-07T19:03:57.858ZMax gen: 196, Max flush: 343
215532024-02-07T19:03:57.858ZError: Difference in extent metadata found!
215542024-02-07T19:03:57.858ZA Difference in extent metadata is expected here
215552024-02-07T19:03:57.858ZOn loop 98
215562024-02-07T19:03:57.859Z
215572024-02-07T19:03:57.862ZVerifying data now
215582024-02-07T19:03:58.380ZNo change after verify
215592024-02-07T19:03:58.380ZLoop: 98 Downstairs dump after verify (and repair):
215602024-02-07T19:03:58.395ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215612024-02-07T19:03:58.395Z 0 000-019 190 190 190 334 334 334 F F F
215622024-02-07T19:03:58.395Z 1 020-039 190 190 190 334 334 334 F F F
215632024-02-07T19:03:58.395Z 2 040-059 196 196 196 343 343 343 F F F
215642024-02-07T19:03:58.395Z 3 060-079 192 192 192 335 335 335 F F F
215652024-02-07T19:03:58.395Z 4 080-099 189 189 189 333 333 333 F F F
215662024-02-07T19:03:58.395Z 5 100-119 194 194 194 341 341 341 F F F
215672024-02-07T19:03:58.395Z 6 120-139 196 196 196 343 343 343 F F F
215682024-02-07T19:03:58.395Z 7 140-159 183 183 183 324 324 324 F F F
215692024-02-07T19:03:58.395Z 8 160-179 186 186 186 328 328 328 F F F
215702024-02-07T19:03:58.395Z 9 180-199 194 194 194 340 340 340 F F F
215712024-02-07T19:03:58.399Z 10 200-219 197 197 197 344 344 344 F F F
215722024-02-07T19:03:58.399Z 11 220-239 188 188 188 332 332 332 F F F
215732024-02-07T19:03:58.399Z 12 240-259 193 193 193 339 339 339 F F F
215742024-02-07T19:03:58.399Z 13 260-279 197 197 197 344 344 344 F F F
215752024-02-07T19:03:58.399Z 14 280-299 197 197 197 344 344 344 F F F
215762024-02-07T19:03:58.399Z 15 300-319 182 182 182 323 323 323 F F F
215772024-02-07T19:03:58.399Z 16 320-339 193 193 193 339 339 339 F F F
215782024-02-07T19:03:58.399Z 17 340-359 197 197 197 344 344 344 F F F
215792024-02-07T19:03:58.399Z 18 360-379 197 197 197 344 344 344 F F F
215802024-02-07T19:03:58.399Z 19 380-399 193 193 193 339 339 339 F F F
215812024-02-07T19:03:58.399Z 20 400-419 197 197 197 344 344 344 F F F
215822024-02-07T19:03:58.399Z 21 420-439 197 197 197 344 344 344 F F F
215832024-02-07T19:03:58.399Z 22 440-459 194 194 194 341 341 341 F F F
215842024-02-07T19:03:58.399Z 23 460-479 190 190 190 334 334 334 F F F
215852024-02-07T19:03:58.399Z 24 480-499 187 187 187 330 330 330 F F F
215862024-02-07T19:03:58.399Z 25 500-519 196 196 196 343 343 343 F F F
215872024-02-07T19:03:58.400Z 26 520-539 187 187 187 330 330 330 F F F
215882024-02-07T19:03:58.400Z 27 540-559 194 194 194 341 341 341 F F F
215892024-02-07T19:03:58.400Z 28 560-579 194 194 194 341 341 341 F F F
215902024-02-07T19:03:58.400Z 29 580-599 190 190 190 334 334 334 F F F
215912024-02-07T19:03:58.400ZMax gen: 197, Max flush: 344
215922024-02-07T19:03:58.400Z
215932024-02-07T19:03:58.400ZBegin loop 99
215942024-02-07T19:03:58.400ZDownstairs to restart: 2
215952024-02-07T19:03:58.416Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605233003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927}
215962024-02-07T19:03:58.416Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"49aa1f31c117032612b377e7cd7453d4ff2c50d7\",\n git_commit_timestamp: \"2024-02-07T18:13:13.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605487299Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927}
215972024-02-07T19:03:58.424Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605524864Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927}
215982024-02-07T19:03:58.424ZCrucible runtime is spawned
215992024-02-07T19:03:58.424Z{"msg":"Crucible b4bb66cc-5c9a-4752-bc0d-6264b868118a has session id: e7d08229-c18a-41e2-86a3-fbb5de324f6b","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605567408Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216002024-02-07T19:03:58.424Z{"msg":"Upstairs opts: Upstairs UUID: b4bb66cc-5c9a-4752-bc0d-6264b868118a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605600324Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216012024-02-07T19:03:58.424Z{"msg":"Crucible stats registered with UUID: b4bb66cc-5c9a-4752-bc0d-6264b868118a","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605643538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216022024-02-07T19:03:58.424Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605677033Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927}
216032024-02-07T19:03:58.424Z{"msg":"b4bb66cc-5c9a-4752-bc0d-6264b868118a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605804086Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216042024-02-07T19:03:58.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605837571Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216052024-02-07T19:03:58.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605864008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216062024-02-07T19:03:58.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605897273Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216072024-02-07T19:03:58.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605925869Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216082024-02-07T19:03:58.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605952446Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216092024-02-07T19:03:58.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.605980152Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216102024-02-07T19:03:58.424Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606027946Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216112024-02-07T19:03:58.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606056432Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216122024-02-07T19:03:58.425Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606095177Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216132024-02-07T19:03:58.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606118643Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216142024-02-07T19:03:58.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606303568Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216152024-02-07T19:03:58.425Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:57.606379008Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216162024-02-07T19:03:58.425Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:57.606480884Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216172024-02-07T19:03:58.425Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:57.606600448Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216182024-02-07T19:03:58.425Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606650331Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216192024-02-07T19:03:58.425Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606695265Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216202024-02-07T19:03:58.425Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606725351Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216212024-02-07T19:03:58.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606768646Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216222024-02-07T19:03:58.425Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606795162Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216232024-02-07T19:03:58.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606863483Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216242024-02-07T19:03:58.425Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.606915526Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216252024-02-07T19:03:58.425Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.607386012Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216262024-02-07T19:03:58.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.607495427Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216272024-02-07T19:03:58.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.607929529Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216282024-02-07T19:03:58.425Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.607965853Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216292024-02-07T19:03:58.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.607982091Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216302024-02-07T19:03:58.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.608028455Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216312024-02-07T19:03:58.425Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.608047002Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216322024-02-07T19:03:58.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:57.60806863Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216332024-02-07T19:03:59.422Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.607819773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216342024-02-07T19:03:59.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.607964773Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216352024-02-07T19:03:59.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.609076632Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216362024-02-07T19:03:59.971Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.609119607Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216372024-02-07T19:03:59.971Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.609695359Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216382024-02-07T19:03:59.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610309366Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216392024-02-07T19:03:59.971Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.61035428Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216402024-02-07T19:03:59.971Z{"msg":"[0]R flush_numbers[0..12]: [334, 334, 343, 335, 333, 341, 343, 324, 328, 340, 344, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610375677Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216412024-02-07T19:03:59.971Z{"msg":"[0]R generation[0..12]: [190, 190, 196, 192, 189, 194, 196, 183, 186, 194, 197, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610404413Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216422024-02-07T19:03:59.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.61042322Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216432024-02-07T19:03:59.971Z{"msg":"[1]R flush_numbers[0..12]: [334, 334, 343, 335, 333, 341, 343, 324, 328, 340, 344, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610450147Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216442024-02-07T19:03:59.971Z{"msg":"[1]R generation[0..12]: [190, 190, 196, 192, 189, 194, 196, 183, 186, 194, 197, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610468874Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216452024-02-07T19:03:59.971Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610495251Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216462024-02-07T19:03:59.971Z{"msg":"[2]R flush_numbers[0..12]: [334, 334, 343, 335, 333, 341, 343, 324, 328, 340, 344, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610514838Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216472024-02-07T19:03:59.971Z{"msg":"[2]R generation[0..12]: [190, 190, 196, 192, 189, 194, 196, 183, 186, 194, 197, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610539235Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216482024-02-07T19:03:59.971Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
216492024-02-07T19:03:59.971Z,"time":"2024-02-07T19:03:58.610554003Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216502024-02-07T19:03:59.971Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.61057642Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216512024-02-07T19:03:59.971Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610592178Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216522024-02-07T19:03:59.971Z{"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610612285Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216532024-02-07T19:03:59.971Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610626263Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216542024-02-07T19:03:59.971Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.61064664Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216552024-02-07T19:03:59.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610659689Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216562024-02-07T19:03:59.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610672867Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216572024-02-07T19:03:59.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610693154Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216582024-02-07T19:03:59.972Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610706682Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216592024-02-07T19:03:59.972Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610725349Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216602024-02-07T19:03:59.972Z{"msg":"b4bb66cc-5c9a-4752-bc0d-6264b868118a is now active with session: e7d08229-c18a-41e2-86a3-fbb5de324f6b","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610738538Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216612024-02-07T19:03:59.972Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610757325Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927,"session_id":"e7d08229-c18a-41e2-86a3-fbb5de324f6b"}
216622024-02-07T19:03:59.972Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-07T19:03:58.610770394Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927}
216632024-02-07T19:03:59.972ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
216642024-02-07T19:03:59.972ZLoading write count information from file "/tmp/test_repair_verify.data"
216652024-02-07T19:03:59.972ZRun Repair workload
216662024-02-07T19:03:59.972Z01/30 Write block 334 len 24576 data: 19 18 18 16 18 21
216672024-02-07T19:03:59.972Z02/30 Write block 70 len 24576 data: 14 13 12 12 11 12
216682024-02-07T19:03:59.972Z03/30 Read block 221 len 16384
216692024-02-07T19:03:59.979Z04/30 Write block 362 len 16384 data: 18 18 13 12
216702024-02-07T19:03:59.979Z05/30 Read block 287 len 8192
216712024-02-07T19:03:59.979Z06/30 Read block 102 len 28672
216722024-02-07T19:03:59.979Z07/30 Write block 262 len 28672 data: 15 19 22 18 18 19 15
216732024-02-07T19:03:59.979Z08/30 Write block 233 len 4096 data: 16
216742024-02-07T19:03:59.979Z09/30 Write block 185 len 16384 data: 13 12 14 14
216752024-02-07T19:03:59.979Z10/30 Read block 70 len 12288
216762024-02-07T19:03:59.979Z11/30 Write block 519 len 16384 data: 18 18 16 11
216772024-02-07T19:03:59.979Z12/30 Read block 217 len 12288
216782024-02-07T19:03:59.979Z13/30 Read block 372 len 24576
216792024-02-07T19:03:59.979Z14/30 Write block 534 len 40960 data: 15 18 18 20 18 17 18 19 18 12
216802024-02-07T19:03:59.979Z15/30 Read block 566 len 36864
216812024-02-07T19:03:59.979Z16/30 Read block 240 len 12288
216822024-02-07T19:03:59.979Z17/30 Write block 502 len 4096 data: 15
216832024-02-07T19:03:59.979Z18/30 Write block 204 len 8192 data: 14 16
216842024-02-07T19:03:59.979Z19/30 Write block 28 len 36864 data: 16 20 19 20 22 21 23 21 21
216852024-02-07T19:03:59.979Z20/30 Write block 275 len 40960 data: 14 14 14 14 13 14 16 15 13 19
216862024-02-07T19:03:59.980Z21/30 Read block 219 len 36864
216872024-02-07T19:03:59.980Z22/30 Read block 244 len 28672
216882024-02-07T19:03:59.980Z23/30 Write block 344 len 40960 data: 17 20 22 22 19 18 20 19 14 12
216892024-02-07T19:03:59.980Z24/30 Write block 477 len 24576 data: 13 14 13 12 12 12
216902024-02-07T19:03:59.980Z25/30 Read block 324 len 40960
216912024-02-07T19:03:59.980Z26/30 Flush
216922024-02-07T19:03:59.980Z27/30 Write block 431 len 32768 data: 18 17 14 11 11 12 15 11
216932024-02-07T19:03:59.980Z28/30 Write block 220 len 24576 data: 11 11 11 14 15 12
216942024-02-07T19:03:59.980Z29/30 Write block 581 len 32768 data: 14 17 15 17 19 19 14 17
216952024-02-07T19:03:59.980Z30/30 Read block 7 len 24576
216962024-02-07T19:03:59.980Z----------------------------------------------------------------
216972024-02-07T19:03:59.980Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
216982024-02-07T19:03:59.980ZStates: Active Active Active
216992024-02-07T19:03:59.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
217002024-02-07T19:03:59.980Z 1 Acked 1000 Write 6 Done Done Done false
217012024-02-07T19:03:59.980Z 2 Acked 1001 Write 6 Done Done Done false
217022024-02-07T19:03:59.980Z 3 Acked 1002 Read 4 Done Done Done false
217032024-02-07T19:03:59.980Z 4 Acked 1003 Write 4 Done Done Sent false
217042024-02-07T19:03:59.980Z 5 Acked 1004 Read 2 Done Done Sent false
217052024-02-07T19:03:59.980Z 6 Acked 1005 Read 7 Done Done Sent false
217062024-02-07T19:03:59.980Z 7 Acked 1006 Write 7 Done Done Sent false
217072024-02-07T19:03:59.980Z 8 Acked 1007 Write 1 Done Done Sent false
217082024-02-07T19:03:59.980Z 9 Acked 1008 Write 4 Done Done Sent false
217092024-02-07T19:03:59.980Z 10 Acked 1009 Read 3 Done Done Sent false
217102024-02-07T19:03:59.981Z 11 Acked 1010 Write 4 Done Done Sent false
217112024-02-07T19:03:59.981Z 12 Acked 1011 Read 3 Done Done Sent false
217122024-02-07T19:03:59.981Z 13 Acked 1012 Read 6 Done Done Sent false
217132024-02-07T19:03:59.981Z 14 Acked 1013 Write 10 Done Done Sent false
217142024-02-07T19:03:59.981Z 15 Acked 1014 Read 9 Done Done Sent false
217152024-02-07T19:03:59.981Z 16 Acked 1015 Read 3 Done Done Sent false
217162024-02-07T19:03:59.981Z 17 Acked 1016 Write 1 Done Done Sent false
217172024-02-07T19:03:59.981Z 18 Acked 1017 Write 2 Done Done Sent false
217182024-02-07T19:03:59.981Z 19 Acked 1018 Write 9 Done Done Sent false
217192024-02-07T19:03:59.981Z 20 Acked 1019 Write 10 Done Done Sent false
217202024-02-07T19:03:59.981Z 21 Acked 1020 Read 9 Done Done Sent false
217212024-02-07T19:03:59.981Z 22 Acked 1021 Read 7 Done Done Sent false
217222024-02-07T19:03:59.981Z 23 Acked 1022 Write 10 Done Done Sent false
217232024-02-07T19:03:59.981Z 24 Acked 1023 Write 6 Done Done Sent false
217242024-02-07T19:03:59.981Z 25 Acked 1024 Read 10 Done Done Sent false
217252024-02-07T19:03:59.981Z 26 Acked 1025 Flush 0 Done Done Sent false
217262024-02-07T19:03:59.981Z 27 Acked 1026 Write 8 Done Done Sent false
217272024-02-07T19:03:59.981Z 28 Acked 1027 Write 6 Done Done Sent false
217282024-02-07T19:03:59.981Z 29 Acked 1028 Write 8 Done Done Sent false
217292024-02-07T19:03:59.981Z 30 Acked 1029 Read 6 Done Done Sent false
217302024-02-07T19:03:59.981Z STATES DS:0 DS:1 DS:2 TOTAL
217312024-02-07T19:03:59.981Z New 0 0 0 0
217322024-02-07T19:03:59.981Z Sent 0 0 27 27
217332024-02-07T19:03:59.982Z Done 30 30 3 63
217342024-02-07T19:03:59.982Z Skipped 0 0 0 0
217352024-02-07T19:03:59.982Z Error 0 0 0 0
217362024-02-07T19:03:59.982ZLast Flush: 1025 1025 0
217372024-02-07T19:03:59.982ZDownstairs last five completed:
217382024-02-07T19:03:59.982ZUpstairs last five completed: 30 29 28 27 26
217392024-02-07T19:03:59.982Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-07T19:03:58.644722122Z","hostname":"ip-10-150-1-187.us-west-2.compute.internal","pid":1927}
217402024-02-07T19:03:59.982ZWrote out file "/tmp/test_repair_verify.data"
217412024-02-07T19:03:59.982Z
217422024-02-07T19:03:59.982ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
217432024-02-07T19:03:59.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217442024-02-07T19:03:59.982Z 0 000-019 190 190 190 334 334 334 F F F
217452024-02-07T19:03:59.982Z 1 020-039 198 198 190 345 345 334 F F F <---
217462024-02-07T19:03:59.982Z 2 040-059 196 196 196 343 343 343 F F F
217472024-02-07T19:03:59.982Z 3 060-079 198 198 192 345 345 335 F F T <---
217482024-02-07T19:03:59.982Z 4 080-099 189 189 189 333 333 333 F F F
217492024-02-07T19:03:59.982Z 5 100-119 194 194 194 341 341 341 F F F
217502024-02-07T19:03:59.982Z 6 120-139 196 196 196 343 343 343 F F F
217512024-02-07T19:03:59.982Z 7 140-159 183 183 183 324 324 324 F F F
217522024-02-07T19:03:59.982Z 8 160-179 186 186 186 328 328 328 F F F
217532024-02-07T19:03:59.982Z 9 180-199 198 198 194 345 345 340 F F F <---
217542024-02-07T19:03:59.982Z 10 200-219 198 198 197 345 345 344 F F F <---
217552024-02-07T19:03:59.982Z 11 220-239 198 198 188 345 345 332 T T F <---
217562024-02-07T19:03:59.982Z 12 240-259 193 193 193 339 339 339 F F F
217572024-02-07T19:03:59.983Z 13 260-279 198 198 197 345 345 344 F F F <---
217582024-02-07T19:03:59.983Z 14 280-299 198 198 197 345 345 344 F F F <---
217592024-02-07T19:03:59.983Z 15 300-319 182 182 182 323 323 323 F F F
217602024-02-07T19:03:59.983Z 16 320-339 198 198 193 345 345 339 F F T <---
217612024-02-07T19:03:59.983Z 17 340-359 198 198 197 345 345 344 F F F <---
217622024-02-07T19:03:59.983Z 18 360-379 198 198 197 345 345 344 F F F <---
217632024-02-07T19:03:59.983Z 19 380-399 193 193 193 339 339 339 F F F
217642024-02-07T19:03:59.983Z 20 400-419 197 197 197 344 344 344 F F F
217652024-02-07T19:03:59.983Z 21 420-439 197 197 197 344 344 344 T T F <---
217662024-02-07T19:03:59.983Z 22 440-459 194 194 194 341 341 341 F F F
217672024-02-07T19:03:59.983Z 23 460-479 198 198 190 345 345 334 F F F <---
217682024-02-07T19:03:59.983Z 24 480-499 198 198 187 345 345 330 F F F <---
217692024-02-07T19:03:59.989Z 25 500-519 198 198 196 345 345 343 F F F <---
217702024-02-07T19:03:59.989Z 26 520-539 198 198 187 345 345 330 F F F <---
217712024-02-07T19:03:59.989Z 27 540-559 198 198 194 345 345 341 F F F <---
217722024-02-07T19:03:59.989Z 28 560-579 194 194 194 341 341 341 F F F
217732024-02-07T19:03:59.989Z 29 580-599 190 190 190 334 334 334 T T F <---
217742024-02-07T19:03:59.989ZMax gen: 198, Max flush: 345
217752024-02-07T19:03:59.989ZError: Difference in extent metadata found!
217762024-02-07T19:03:59.989ZA Difference in extent metadata is expected here
217772024-02-07T19:03:59.989ZOn loop 99
217782024-02-07T19:03:59.989Z
217792024-02-07T19:03:59.989ZVerifying data now
217802024-02-07T19:04:04.279ZNo change after verify
217812024-02-07T19:04:04.280ZLoop: 99 Downstairs dump after verify (and repair):
217822024-02-07T19:04:04.295ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217832024-02-07T19:04:04.296Z 0 000-019 190 190 190 334 334 334 F F F
217842024-02-07T19:04:04.296Z 1 020-039 198 198 198 345 345 345 F F F
217852024-02-07T19:04:04.296Z 2 040-059 196 196 196 343 343 343 F F F
217862024-02-07T19:04:04.296Z 3 060-079 198 198 198 345 345 345 F F F
217872024-02-07T19:04:04.296Z 4 080-099 189 189 189 333 333 333 F F F
217882024-02-07T19:04:04.296Z 5 100-119 194 194 194 341 341 341 F F F
217892024-02-07T19:04:04.296Z 6 120-139 196 196 196 343 343 343 F F F
217902024-02-07T19:04:04.296Z 7 140-159 183 183 183 324 324 324 F F F
217912024-02-07T19:04:04.296Z 8 160-179 186 186 186 328 328 328 F F F
217922024-02-07T19:04:04.296Z 9 180-199 198 198 198 345 345 345 F F F
217932024-02-07T19:04:04.296Z 10 200-219 198 198 198 345 345 345 F F F
217942024-02-07T19:04:04.297Z 11 220-239 199 199 199 346 346 346 F F F
217952024-02-07T19:04:04.301Z 12 240-259 193 193 193 339 339 339 F F F
217962024-02-07T19:04:04.301Z 13 260-279 198 198 198 345 345 345 F F F
217972024-02-07T19:04:04.301Z 14 280-299 198 198 198 345 345 345 F F F
217982024-02-07T19:04:04.301Z 15 300-319 182 182 182 323 323 323 F F F
217992024-02-07T19:04:04.301Z 16 320-339 198 198 198 345 345 345 F F F
218002024-02-07T19:04:04.301Z 17 340-359 198 198 198 345 345 345 F F F
218012024-02-07T19:04:04.301Z 18 360-379 198 198 198 345 345 345 F F F
218022024-02-07T19:04:04.301Z 19 380-399 193 193 193 339 339 339 F F F
218032024-02-07T19:04:04.301Z 20 400-419 197 197 197 344 344 344 F F F
218042024-02-07T19:04:04.301Z 21 420-439 199 199 199 346 346 346 F F F
218052024-02-07T19:04:04.301Z 22 440-459 194 194 194 341 341 341 F F F
218062024-02-07T19:04:04.301Z 23 460-479 198 198 198 345 345 345 F F F
218072024-02-07T19:04:04.301Z 24 480-499 198 198 198 345 345 345 F F F
218082024-02-07T19:04:04.301Z 25 500-519 198 198 198 345 345 345 F F F
218092024-02-07T19:04:04.302Z 26 520-539 198 198 198 345 345 345 F F F
218102024-02-07T19:04:04.302Z 27 540-559 198 198 198 345 345 345 F F F
218112024-02-07T19:04:04.302Z 28 560-579 194 194 194 341 341 341 F F F
218122024-02-07T19:04:04.302Z 29 580-599 199 199 199 346 346 346 F F F
218132024-02-07T19:04:04.302ZMax gen: 199, Max flush: 346
218142024-02-07T19:04:04.302Z9:39 Test duration
218152024-02-07T19:04:04.302ZTest completed
218162024-02-07T19:04:04.302Z
218172024-02-07T19:04:04.302Zreal 9:38.562609914
218182024-02-07T19:04:04.302Zuser 1:34.607977155
218192024-02-07T19:04:04.302Zsys 17.281667862
218202024-02-07T19:04:04.302Ztrap 0.085269866
218212024-02-07T19:04:04.302Ztflt 0.060090427
218222024-02-07T19:04:04.302Zdflt 0.054112072
218232024-02-07T19:04:04.302Zkflt 0.003217481
218242024-02-07T19:04:04.302Zlock 7:30:28.934095025
218252024-02-07T19:04:04.302Zslp 46:34.094839595
218262024-02-07T19:04:04.302Zlat 5.638993318
218272024-02-07T19:04:04.302Zstop 0.048540997
218282024-02-07T19:04:04.302Zprocess exited: duration 587713 ms, exit code 0
 
218292024-02-07T19:04:04.310Zfound 4 output files
218302024-02-07T19:04:04.310Zuploading: /tmp/test_repair_ds8810.txt (42008 bytes)
218312024-02-07T19:04:05.332Zuploaded: /tmp/test_repair_ds8810.txt
218322024-02-07T19:04:05.335Zuploading: /tmp/test_repair_ds8820.txt (57218 bytes)
218332024-02-07T19:04:06.353Zuploaded: /tmp/test_repair_ds8820.txt
218342024-02-07T19:04:06.360Zuploading: /tmp/test_repair_ds8830.txt (25769 bytes)
218352024-02-07T19:04:07.386Zuploaded: /tmp/test_repair_ds8830.txt
218362024-02-07T19:04:07.824Zuploading: /tmp/test_repair_out.txt (108748 bytes)
218372024-02-07T19:04:08.447Zuploaded: /tmp/test_repair_out.txt